@ganju/utils 0.0.15 → 0.0.17

package/src/customCodeToken.ts

@@ -178,6 +178,80 @@ export const verifyCustomCodeToken = async (
   };
 };
 
+// Names the function one dispatch runs.
+//
+// The tool token names the artifact and version but not the function: one
+// script serves every function, and it would otherwise say itself which one
+// is calling. What a function may do with the database (its manifest's
+// `data`) needs a name the script can't choose, so the dispatcher signs one
+// per call and passes it in with the input; the script only hands it back.
+// Signed with a key derived from the same secret, so a call token is never
+// a tool token or the other way round.
+export interface CustomCodeCallPayload {
+  v: string;
+  artifactId: string;
+  versionId: string;
+  tool: string;
+  exp: number;
+}
+
+const callSecret = (secret: string) => `${secret}:call`;
+
+export const mintCustomCodeCallToken = async (
+  payload: Omit<CustomCodeCallPayload, 'v' | 'exp'> & { ttlMs: number },
+  secret: string,
+  issuedAt: number = Date.now()
+): Promise<string> => {
+  const body: CustomCodeCallPayload = {
+    v: constants.CUSTOM_CODE_TOKEN_VERSION,
+    artifactId: payload.artifactId,
+    versionId: payload.versionId,
+    tool: payload.tool,
+    exp: Math.ceil((issuedAt + payload.ttlMs) / 1000)
+  };
+  const encodedPayload = toBase64Url(
+    bytesToBase64(encoder.encode(JSON.stringify(body)))
+  );
+  const signature = await signPayload(encodedPayload, callSecret(secret));
+  return `${encodedPayload}.${signature}`;
+};
+
+export const verifyCustomCodeCallToken = async (
+  token: string,
+  secret: string,
+  now: number = Date.now()
+): Promise<CustomCodeCallPayload | null> => {
+  const separator = token.lastIndexOf('.');
+  if (separator <= 0) return null;
+  const encodedPayload = token.slice(0, separator);
+  try {
+    const key = await importKey(callSecret(secret));
+    const valid = await crypto.subtle.verify(
+      'HMAC',
+      key,
+      new Uint8Array(base64ToBytes(fromBase64Url(token.slice(separator + 1)))),
+      encoder.encode(encodedPayload)
+    );
+    if (!valid) return null;
+    const candidate = JSON.parse(
+      new TextDecoder().decode(base64ToBytes(fromBase64Url(encodedPayload)))
+    ) as Partial<CustomCodeCallPayload>;
+    if (
+      candidate.v !== constants.CUSTOM_CODE_TOKEN_VERSION ||
+      typeof candidate.artifactId !== 'string' ||
+      typeof candidate.versionId !== 'string' ||
+      typeof candidate.tool !== 'string' ||
+      typeof candidate.exp !== 'number' ||
+      candidate.exp * 1000 <= now
+    ) {
+      return null;
+    }
+    return candidate as CustomCodeCallPayload;
+  } catch {
+    return null;
+  }
+};
+
 /**
  * The legacy dispatch-namespace script name for an artifact: `artifact_<id>`.
  *

package/src/data.ts

@@ -0,0 +1,108 @@
+// Entry point for `@ganju/utils/data`: everything a project's database and
+// its callers share, without pulling in the rest of the package.
+export * from './dataTypes';
+export {
+  DATA_IDENTIFIER_PATTERN,
+  DATA_SYSTEM_COLUMNS,
+  DATA_HIDDEN_COLUMNS,
+  identifierIssue,
+  isValidIdentifier
+} from './dataIdentifiers';
+export {
+  createTableSql,
+  columnDefinitionSql,
+  desiredIndexes,
+  desiredTriggers,
+  ftsCreateSql,
+  ftsTableName,
+  searchableColumns,
+  sqlLiteral
+} from './dataDdl';
+export {
+  chunkRows,
+  rowsPerInsert,
+  ftsMatch,
+  keysetCondition,
+  encodeCursor,
+  decodeCursor,
+  isSensitiveWrite,
+  tidyArgs,
+  zoned
+} from './dataCompiler';
+export type { SqlValue, KeysetCursor, SortKey } from './dataCompiler';
+export {
+  INTERNAL_MIGRATIONS,
+  INTERNAL_SCHEMA_VERSION,
+  hasInternalTables,
+  getMeta,
+  setMeta,
+  migrateInternal,
+  loadSchema,
+  toTableInput,
+  normaliseColumn,
+  translateSqliteError,
+  describe,
+  query,
+  write,
+  rowHistory,
+  pruneHistory,
+  deleteImpact,
+  createTable,
+  updateTable,
+  dropTable,
+  addColumn,
+  updateColumn,
+  dropColumn,
+  columnChangeNeedsRebuild,
+  dropColumnNeedsRebuild,
+  readTableCopy,
+  clearFailedTableCopy,
+  copyTableStep,
+  writeReachesCopy,
+  assertActor,
+  assertWritable,
+  exportRows
+} from './dataEngine';
+export type { SqlRow, SqlRunner } from './dataEngine';
+export {
+  DATA_QUERY_PARAM_TYPES,
+  DATA_MAX_QUERIES,
+  DATA_MAX_QUERY_SQL,
+  DATA_MAX_QUERY_PARAMS,
+  DATA_DEFAULT_QUERY_ROWS,
+  compileQuerySql,
+  checkQueryPlan,
+  listQueries,
+  queriesFor,
+  saveQuery,
+  dropQuery,
+  runQuery
+} from './dataQueries';
+export type {
+  DataQuery,
+  DataQueryInput,
+  DataQueryParam,
+  DataQueryParamType
+} from './dataQueries';
+export type {
+  ProjectDatabaseRpc,
+  DatabaseInfo,
+  ExportFile,
+  ExportFormat,
+  ExportState,
+  RestoreResult
+} from './dataRpc';
+export {
+  DATA_TOOL_DESCRIBE,
+  DATA_TOOL_QUERY,
+  DATA_TOOL_WRITE,
+  DATA_TOOL_ADMIN,
+  DATA_TOOL_NAMES,
+  DATA_TOOL_TITLES,
+  DATA_TOOL_MAX_LIMIT,
+  DATA_ADMIN_ACTIONS,
+  hasDataToolTables,
+  dataToolDefinitions,
+  dataAdminToolDefinition
+} from './dataTools';
+export type { DataToolDefinition, DataAdminAction } from './dataTools';

package/src/dataApi.ts

@@ -0,0 +1,479 @@
+// The Data API's routes, as one list: apps/api registers its data routes from
+// it, and the OpenAPI file is generated from it, so the documented API and the
+// served one can't drift apart. Entry point `@ganju/utils/dataApi`.
+import { z } from 'zod';
+
+import { Schema } from './schema';
+
+export const DATA_API_BASE =
+  '/organization/:organizationId/project/:projectId/artifact/data';
+
+export type DataApiMethod = 'get' | 'post' | 'patch' | 'delete';
+
+export interface DataApiRoute {
+  method: DataApiMethod;
+  // Relative to DATA_API_BASE, in Hono's `:param` form.
+  path: string;
+  summary: string;
+  description?: string;
+  tag: 'Database' | 'Tables' | 'Rows' | 'Queries' | 'History and export';
+  body?: z.ZodType;
+  query?: z.ZodObject;
+  // What a 200/201 answers with, in a sentence.
+  returns: string;
+  status?: 200 | 201;
+  // Answers with a file instead of JSON.
+  file?: boolean;
+}
+
+const {
+  DATA_DATABASE_CREATE,
+  DATA_DATABASE_DELETE,
+  DATA_DATABASE_PATCH,
+  DATA_TABLE_INPUT,
+  DATA_TABLE_PATCH,
+  DATA_COLUMN_INPUT,
+  DATA_COLUMN_PATCH,
+  DATA_TABLE_DROP_BODY,
+  DATA_ROW_LIST_QUERY,
+  DATA_ROW_GET_QUERY,
+  DATA_ROW_CREATE_BODY,
+  DATA_ROW_UPDATE_BODY,
+  DATA_ROW_DELETE_BODY,
+  DATA_DELETE_PREVIEW_BODY,
+  DATA_WRITE_BODY,
+  DATA_ROW_HISTORY_QUERY,
+  DATA_RESTORE_POINT_BODY,
+  DATA_EXPORT_BODY,
+  DATA_EXPORT_QUERY,
+  DATA_EXPORT_FILE_QUERY,
+  DATA_CHANGE_LIST_QUERY,
+  DATA_QUERY_INPUT,
+  DATA_QUERY_RUN_BODY
+} = Schema;
+
+export const DATA_API_ROUTES = {
+  getDatabase: {
+    method: 'get',
+    path: '',
+    tag: 'Database',
+    summary: "The project's database",
+    description:
+      'Its status and size, and its tables when it is ready. `database` is null when the project has none.',
+    returns: '{ database }'
+  },
+  create: {
+    method: 'post',
+    path: '',
+    tag: 'Database',
+    summary: 'Create the database',
+    description:
+      "Pro and Enterprise. Safe to repeat: a second call returns the same database. `name` defaults to the project's slug. While a deleted database is in its grace period this is refused unless `replaceDeleted` is true: the deleted one can then no longer be restored or exported, and is emptied when its grace period ends.",
+    body: DATA_DATABASE_CREATE.pick({ name: true, replaceDeleted: true }),
+    returns: '{ database }',
+    status: 201
+  },
+  updateDatabase: {
+    method: 'patch',
+    path: '',
+    tag: 'Database',
+    summary: "Change the database's settings",
+    description:
+      '`timeZone`, an IANA zone such as `America/Bogota`, or null for UTC. Datetimes come back in it for callers that send no `timeZone` of their own: the assistant, functions and scripts. Datetimes without an offset are read in it too.',
+    body: DATA_DATABASE_PATCH,
+    returns: '{ database }'
+  },
+  remove: {
+    method: 'delete',
+    path: '',
+    tag: 'Database',
+    summary: 'Delete the database',
+    description:
+      'Needs `confirmName`, the database name typed out. It can be restored for 7 days.',
+    body: DATA_DATABASE_DELETE.pick({ confirmName: true }),
+    returns: '{ database }'
+  },
+  restore: {
+    method: 'post',
+    path: '/restore',
+    tag: 'Database',
+    summary: 'Undo a delete',
+    description: 'Within the 7-day grace period.',
+    returns: '{ database }'
+  },
+  restorePoint: {
+    method: 'post',
+    path: '/restore-point',
+    tag: 'Database',
+    summary: 'Restore to a point in time',
+    description:
+      'Replaces everything with the database as it was at `timestamp` (within 30 days), or at the `bookmark` a checkpoint, a dropped table or column, a type change or an earlier restore answered with. The answer carries `undoBookmark`, which undoes the restore. Calls in flight while it restarts fail.',
+    body: DATA_RESTORE_POINT_BODY,
+    returns: '{ bookmark, undoBookmark }'
+  },
+  checkpoint: {
+    method: 'post',
+    path: '/checkpoint',
+    tag: 'Database',
+    summary: 'Take a restore point',
+    description:
+      "Answers with the `bookmark` of the database as it is right now, which `POST /restore-point` accepts. Take one before a large change, such as replacing a table's rows, to undo exactly that change: a time is not that precise.",
+    returns: '{ bookmark }'
+  },
+  dismissCopy: {
+    method: 'delete',
+    path: '/copy',
+    tag: 'Tables',
+    summary: 'Dismiss a failed table copy',
+    description:
+      'A column change on a table over 100,000 rows copies it in the background; table changes, and writes that would change its rows, wait until it finishes (`copy` on `GET` shows how far it got). A copy that failed left the table as it was and stays in `copy` with its `error` until dismissed, or until the next column change.',
+    returns: '{ copy: null }'
+  },
+  listTables: {
+    method: 'get',
+    path: '/table',
+    tag: 'Tables',
+    summary: 'Tables and their columns',
+    returns: '{ tables }'
+  },
+  createTable: {
+    method: 'post',
+    path: '/table',
+    tag: 'Tables',
+    summary: 'Create a table',
+    body: DATA_TABLE_INPUT,
+    returns: '{ table }',
+    status: 201
+  },
+  updateTable: {
+    method: 'patch',
+    path: '/table/:table',
+    tag: 'Tables',
+    summary: 'Rename a table or change its settings',
+    body: DATA_TABLE_PATCH,
+    returns: '{ table }'
+  },
+  dropTable: {
+    method: 'delete',
+    path: '/table/:table',
+    tag: 'Tables',
+    summary: 'Drop a table',
+    description:
+      'Needs `confirmName`, the table name typed out. The answer carries a `bookmark`: restoring to it brings the table back.',
+    body: DATA_TABLE_DROP_BODY,
+    returns: '{ dropped, bookmark }'
+  },
+  addColumn: {
+    method: 'post',
+    path: '/table/:table/column',
+    tag: 'Tables',
+    summary: 'Add a column',
+    body: DATA_COLUMN_INPUT,
+    returns: '{ table }',
+    status: 201
+  },
+  updateColumn: {
+    method: 'patch',
+    path: '/table/:table/column/:column',
+    tag: 'Tables',
+    summary: 'Rename a column or change it',
+    description:
+      'A type change copies the table and lists the first rows that would not fit. When it copies, the answer carries a `bookmark` from just before. Over 100,000 rows the copy runs in the background: the answer carries `copy`, the table keeps its old columns until it finishes, and meanwhile table changes, and writes that would change its rows, are refused.',
+    body: DATA_COLUMN_PATCH,
+    returns: '{ table, bookmark, copy }'
+  },
+  dropColumn: {
+    method: 'delete',
+    path: '/table/:table/column/:column',
+    tag: 'Tables',
+    summary: 'Drop a column',
+    description:
+      'The answer carries a `bookmark`: restoring to it brings the column back. Dropping a relation column copies the table, in the background over 100,000 rows, as for a type change.',
+    returns: '{ table, bookmark, copy }'
+  },
+  listRows: {
+    method: 'get',
+    path: '/table/:table/row',
+    tag: 'Rows',
+    summary: 'List, filter and search rows',
+    description:
+      'Up to 500 rows or 1 MB a page; `nextCursor` reads the next. `where` is JSON, e.g. `{"status":"open","price":{"lt":30000}}`. `sort` is a list of columns, `-` for descending. `q` searches the searchable columns.',
+    query: DATA_ROW_LIST_QUERY,
+    returns: '{ table, rows, total?, nextCursor? }'
+  },
+  createRows: {
+    method: 'post',
+    path: '/table/:table/row',
+    tag: 'Rows',
+    summary: 'Insert one row, or up to 500',
+    body: DATA_ROW_CREATE_BODY,
+    returns: '{ ids }',
+    status: 201
+  },
+  getRow: {
+    method: 'get',
+    path: '/table/:table/row/:id',
+    tag: 'Rows',
+    summary: 'One row',
+    query: DATA_ROW_GET_QUERY,
+    returns: '{ row }'
+  },
+  updateRow: {
+    method: 'patch',
+    path: '/table/:table/row/:id',
+    tag: 'Rows',
+    summary: 'Change a row',
+    description:
+      'Send the `version` you read: if the row changed since, the answer is 409 with the current row and who changed it.',
+    body: DATA_ROW_UPDATE_BODY,
+    returns: '{ row }'
+  },
+  deleteRow: {
+    method: 'delete',
+    path: '/table/:table/row/:id',
+    tag: 'Rows',
+    summary: 'Delete a row',
+    description: 'Send the `version` you read, as for a change.',
+    body: DATA_ROW_DELETE_BODY,
+    returns: '{ deleted }'
+  },
+  deletePreview: {
+    method: 'post',
+    path: '/table/:table/delete-preview',
+    tag: 'Rows',
+    summary: 'What deleting rows would do',
+    description:
+      'For `ids`, or every row without them: each relation column pointing at a row the delete removes, with how many rows it would delete (cascade), clear (set null) or that refuse the delete (restrict). Cascades are followed all the way down. `blocked` says a restrict would refuse it. Past 200,000 affected rows `truncated` is true and the counts are lower bounds. Nothing is deleted.',
+    body: DATA_DELETE_PREVIEW_BODY,
+    returns: '{ table, rows, effects, blocked, truncated }'
+  },
+  rowHistory: {
+    method: 'get',
+    path: '/table/:table/row/:id/history',
+    tag: 'History and export',
+    summary: "A row's history",
+    description: 'Newest first, kept 90 days, with who made each change.',
+    query: DATA_ROW_HISTORY_QUERY,
+    returns: '{ entries, nextCursor? }'
+  },
+  write: {
+    method: 'post',
+    path: '/write',
+    tag: 'Rows',
+    summary: 'Several writes, all or nothing',
+    description:
+      'Inserts, updates and deletes across tables in one transaction. A guard (`where` plus `expect`) that fails rolls everything back. A later step uses an earlier single-row insert\'s id as `{ "$ref": <step> }`, in its values or as the `id` it updates or deletes.',
+    body: DATA_WRITE_BODY,
+    returns: '{ results }'
+  },
+  listQueries: {
+    method: 'get',
+    path: '/query',
+    tag: 'Queries',
+    summary: 'Named queries',
+    description:
+      'Each with its SQL, parameters, the tables it reads and its plan (`SCAN orders` means it reads the whole table).',
+    returns: '{ queries }'
+  },
+  saveQuery: {
+    method: 'post',
+    path: '/query',
+    tag: 'Queries',
+    summary: 'Save a named query',
+    description:
+      "One `SELECT` (or `WITH … SELECT`) with named parameters such as `:from`, each declared in `params` with its type. It is compiled, never run, and refused unless it only reads this database's tables. Saving under an existing name replaces it. `assistant: true` lets channel bots run it from data-query.",
+    body: DATA_QUERY_INPUT,
+    returns: '{ query }',
+    status: 201
+  },
+  dropQuery: {
+    method: 'delete',
+    path: '/query/:name',
+    tag: 'Queries',
+    summary: 'Delete a named query',
+    returns: '{ dropped }'
+  },
+  runQuery: {
+    method: 'post',
+    path: '/query/:name/run',
+    tag: 'Queries',
+    summary: 'Run a named query',
+    description:
+      'With its `params`, up to `limit` rows (100 by default, 500 at most) and 1 MB; `truncated` says it stopped early. A query that would scan a table over 200,000 rows is refused.',
+    body: DATA_QUERY_RUN_BODY,
+    returns: '{ query, rows, truncated? }'
+  },
+  startExport: {
+    method: 'post',
+    path: '/export',
+    tag: 'History and export',
+    summary: 'Start an export',
+    description:
+      'Writes a copy in the background: schema.json and one JSONL or CSV file per table. Poll `GET /export` until `status` is `done`. Works during the 7-day grace period after a delete too.',
+    body: DATA_EXPORT_BODY,
+    returns: '{ exportId, status }',
+    status: 201
+  },
+  exportStatus: {
+    method: 'get',
+    path: '/export',
+    tag: 'History and export',
+    summary: 'The latest export',
+    query: DATA_EXPORT_QUERY,
+    returns: '{ export }'
+  },
+  exportFile: {
+    method: 'get',
+    path: '/export/file',
+    tag: 'History and export',
+    summary: "Download one of an export's files",
+    description: "A table's file, or schema.json without `table`.",
+    query: DATA_EXPORT_FILE_QUERY,
+    returns: 'The file',
+    file: true
+  },
+  listChanges: {
+    method: 'get',
+    path: '/change',
+    tag: 'History and export',
+    summary: 'The schema change log',
+    description:
+      'Newest first: tables and columns changed, restores, exports, deletes.',
+    query: DATA_CHANGE_LIST_QUERY,
+    returns: '{ changes, nextCursor? }'
+  }
+} as const satisfies Record<string, DataApiRoute>;
+
+export type DataApiRouteId = keyof typeof DATA_API_ROUTES;
+
+// `/table/:table` → `/table/{table}`
+const openApiPath = (path: string) => path.replace(/:([A-Za-z]+)/g, '{$1}');
+
+const jsonSchema = (schema: z.ZodType) =>
+  z.toJSONSchema(schema, {
+    io: 'input',
+    unrepresentable: 'any',
+    target: 'draft-2020-12'
+  });
+
+const errorResponse = (description: string) => ({
+  description,
+  content: {
+    'application/json': { schema: { $ref: '#/components/schemas/Error' } }
+  }
+});
+
+/**
+ * The OpenAPI 3.1 document for the Data API, served by apps/api at
+ * /openapi/data.json.
+ */
+export const dataOpenApi = (serverUrl: string) => {
+  const paths: Record<string, Record<string, unknown>> = {};
+  for (const [id, route] of Object.entries(DATA_API_ROUTES) as [
+    DataApiRouteId,
+    DataApiRoute
+  ][]) {
+    const fullPath = openApiPath(`${DATA_API_BASE}${route.path}`);
+    const pathParams = [...fullPath.matchAll(/\{([A-Za-z]+)\}/g)].map(m => ({
+      name: m[1],
+      in: 'path',
+      required: true,
+      schema:
+        m[1] === 'id' ? { type: 'integer', minimum: 1 } : { type: 'string' }
+    }));
+    const queryParams = route.query
+      ? Object.entries(route.query.shape).map(([name, field]) => ({
+          name,
+          in: 'query',
+          required: !(field as z.ZodType).safeParse(undefined).success,
+          schema: jsonSchema(field as z.ZodType)
+        }))
+      : [];
+    const status = String(route.status ?? 200);
+    paths[fullPath] ??= {};
+    paths[fullPath][route.method] = {
+      operationId: id,
+      tags: [route.tag],
+      summary: route.summary,
+      ...(route.description ? { description: route.description } : {}),
+      parameters: [...pathParams, ...queryParams],
+      ...(route.body
+        ? {
+            requestBody: {
+              required: true,
+              content: {
+                'application/json': { schema: jsonSchema(route.body) }
+              }
+            }
+          }
+        : {}),
+      responses: {
+        [status]: route.file
+          ? {
+              description: route.returns,
+              content: { 'application/octet-stream': {} }
+            }
+          : {
+              description: route.returns,
+              content: { 'application/json': { schema: { type: 'object' } } }
+            },
+        '400': errorResponse(
+          'The request is not valid; `errors` lists each field.'
+        ),
+        '401': errorResponse('No session or token.'),
+        '402': errorResponse('The plan does not include Data.'),
+        '403': errorResponse(
+          'Not an admin of the project, or the token is for another project.'
+        ),
+        '404': errorResponse('No database, table or row by that name.'),
+        '409': errorResponse(
+          'A conflict: a stale `version` (with `details.current`), a guard that failed, a constraint, or a full database.'
+        ),
+        '429': errorResponse('Too many requests for this token; wait a minute.')
+      }
+    };
+  }
+  return {
+    openapi: '3.1.0',
+    info: {
+      title: 'Ganju Data API',
+      version: '1',
+      description:
+        "Tables, rows and search in a project's database. Authenticate with a personal access token (`ganju token create`) as a bearer token; a token reaches only its own project."
+    },
+    servers: [{ url: serverUrl }],
+    security: [{ bearer: [] }],
+    tags: [
+      { name: 'Database' },
+      { name: 'Tables' },
+      { name: 'Rows' },
+      { name: 'Queries' },
+      { name: 'History and export' }
+    ],
+    paths,
+    components: {
+      securitySchemes: { bearer: { type: 'http', scheme: 'bearer' } },
+      schemas: {
+        Error: {
+          type: 'object',
+          properties: {
+            error: { type: 'string' },
+            code: { type: 'string' },
+            details: { type: 'object' },
+            // A request that didn't parse: one entry per field.
+            errors: {
+              type: 'array',
+              items: {
+                type: 'object',
+                properties: {
+                  path: { type: 'string' },
+                  message: { type: 'string' }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  };
+};