@ganju/utils 0.0.15 → 0.0.16

package/src/dataRpc.ts

@@ -0,0 +1,121 @@
+// The methods a project's database object answers over RPC. apps/data's
+// ProjectDatabase implements it; apps/api, apps/mcp and apps/tool-broker call
+// it through databaseStub. Every method resolves to a DataResult.
+import type {
+  DataCaller,
+  DataDeleteImpact,
+  DataResult,
+  DataSchema,
+  DataTable,
+  DataTableCopy
+} from './dataTypes';
+
+export type ExportFormat = 'jsonl' | 'csv';
+
+export interface ExportFile {
+  // null for schema.json
+  table: string | null;
+  key: string;
+  rows: number;
+  bytes: number;
+}
+
+export interface ExportState {
+  id: string;
+  format: ExportFormat;
+  status: 'running' | 'done' | 'failed';
+  startedAt: string;
+  finishedAt: string | null;
+  actor: string;
+  tables: string[];
+  tableIndex: number;
+  afterId: number;
+  upload: {
+    key: string;
+    uploadId: string;
+    parts: { partNumber: number; etag: string }[];
+    rows: number;
+    bytes: number;
+    // Bytes read but not yet uploaded, kept in R2 between steps: every part
+    // but the last must be exactly the same size.
+    carryKey: string | null;
+  } | null;
+  files: ExportFile[];
+  error: string | null;
+}
+
+export interface DatabaseInfo {
+  artifactId: string | null;
+  name: string | null;
+  initializedAt: string | null;
+  deletedAt: string | null;
+  firstWriteAt: string | null;
+  schemaVersion: number;
+  size: number;
+  tables: { name: string; label: string | null; columns: number }[];
+  // A column change copying a large table in the background: table changes,
+  // and writes that would change its rows, wait while it runs.
+  copy: DataTableCopy | null;
+}
+
+export interface RestoreResult {
+  bookmark: string;
+  undoBookmark: string;
+}
+
+type R<T> = Promise<DataResult<T>>;
+
+export interface ProjectDatabaseRpc {
+  init(args: { artifactId: string; name: string }): R<DatabaseInfo>;
+  info(): R<DatabaseInfo>;
+  rename(args: { name: string }): R<DatabaseInfo>;
+  size(): R<{ bytes: number }>;
+  markDeleted(
+    caller: DataCaller
+  ): R<{ deletedAt: string | null; bookmark: string }>;
+  undelete(caller: DataCaller): R<DatabaseInfo>;
+  purge(): R<{ purged: true; bookmark: string }>;
+  currentBookmark(): R<{ bookmark: string }>;
+  restoreTo(caller: DataCaller, args: { timestamp: number }): R<RestoreResult>;
+  restoreToBookmark(args: { bookmark: string }): R<RestoreResult>;
+  schemaRead(): R<DataSchema>;
+  createTable(caller: DataCaller, input: unknown): R<DataTable>;
+  updateTable(caller: DataCaller, table: string, patch: unknown): R<DataTable>;
+  dropTable(
+    caller: DataCaller,
+    table: string
+  ): R<{ dropped: string; bookmark: string }>;
+  addColumn(caller: DataCaller, table: string, input: unknown): R<DataTable>;
+  updateColumn(
+    caller: DataCaller,
+    table: string,
+    column: string,
+    patch: unknown
+  ): R<{ table: DataTable; bookmark: string | null; copy: DataTableCopy | null }>;
+  dropColumn(
+    caller: DataCaller,
+    table: string,
+    column: string
+  ): R<{ table: DataTable; bookmark: string; copy: DataTableCopy | null }>;
+  // Forgets a failed background copy once the owner has seen why.
+  dismissCopy(caller: DataCaller): R<{ copy: null }>;
+  describe(caller: DataCaller, args?: unknown): R<unknown>;
+  query(caller: DataCaller, args: unknown): R<unknown>;
+  write(
+    caller: DataCaller,
+    args: unknown
+  ): R<{ ok: true; results: unknown[] }>;
+  deleteImpact(
+    caller: DataCaller,
+    args: { table: string; ids?: number[] }
+  ): R<DataDeleteImpact>;
+  history(
+    caller: DataCaller,
+    args: { table: string; id: number; limit?: number; cursor?: number }
+  ): R<unknown>;
+  exportStart(
+    caller: DataCaller,
+    args?: { format?: ExportFormat }
+  ): R<{ exportId: string; status: ExportState['status'] }>;
+  exportStatus(args?: { exportId?: string }): R<ExportState>;
+}

package/src/dataTypes.ts

@@ -0,0 +1,312 @@
+// The shapes and limits shared by a project's database object (apps/data),
+// the workers that call it, and the tests. Pure: nothing here touches SQLite.
+
+export const DATA_COLUMN_TYPES = [
+  'text',
+  'long_text',
+  'integer',
+  'decimal',
+  'boolean',
+  'date',
+  'datetime',
+  'enum',
+  'json',
+  'email',
+  'url',
+  'phone',
+  'relation',
+  'file'
+] as const;
+export type DataColumnType = (typeof DATA_COLUMN_TYPES)[number];
+
+// What the assistant may do with a table when the caller is restricted (a
+// channel bot). Owners, the dashboard, the CLI and functions see every table.
+export const DATA_ACCESS_LEVELS = ['none', 'read', 'add', 'full'] as const;
+export type DataAccess = (typeof DATA_ACCESS_LEVELS)[number];
+export const DATA_ACCESS_RANK: Record<DataAccess, number> = {
+  none: 0,
+  read: 1,
+  add: 2,
+  full: 3
+};
+
+export const DATA_ON_DELETE = ['restrict', 'cascade', 'set_null'] as const;
+export type DataOnDelete = (typeof DATA_ON_DELETE)[number];
+
+export type DataDefaultValue = string | number | boolean | null;
+
+// A column as the owner defines it: the dashboard's form, `ganju.data.json`
+// and the table-create body all use this shape.
+export interface DataColumnInput {
+  name: string;
+  type: DataColumnType;
+  label?: string | null;
+  description?: string | null;
+  required?: boolean;
+  unique?: boolean;
+  filterable?: boolean;
+  searchable?: boolean;
+  personal?: boolean;
+  // enum
+  values?: string[];
+  // relation: the table it points at, and what deleting that row does
+  table?: string;
+  onDelete?: DataOnDelete;
+  // integer / decimal: the lowest value the column may hold
+  min?: number | null;
+  default?: DataDefaultValue;
+}
+
+export interface DataTableInput {
+  name: string;
+  label?: string | null;
+  description?: string | null;
+  displayColumn?: string | null;
+  access?: DataAccess;
+  customerScoped?: boolean;
+  history?: boolean;
+  columns: DataColumnInput[];
+}
+
+// A column as the database object stores it: the input plus its stable id,
+// which names its indexes and constraints so a rename never orphans them.
+export interface DataColumn {
+  id: number;
+  name: string;
+  type: DataColumnType;
+  label: string | null;
+  description: string | null;
+  required: boolean;
+  unique: boolean;
+  filterable: boolean;
+  searchable: boolean;
+  personal: boolean;
+  values: string[] | null;
+  table: string | null;
+  onDelete: DataOnDelete | null;
+  min: number | null;
+  default: DataDefaultValue | undefined;
+  position: number;
+}
+
+export interface DataTable {
+  id: number;
+  name: string;
+  label: string | null;
+  description: string | null;
+  displayColumn: string | null;
+  access: DataAccess;
+  customerScoped: boolean;
+  history: boolean;
+  position: number;
+  columns: DataColumn[];
+}
+
+export interface DataSchema {
+  tables: DataTable[];
+}
+
+// Who is calling and under which rules. `actor` is recorded on every row a
+// call writes; the storage layer refuses a write without one.
+export interface DataCaller {
+  // user:<id> · cli:<tokenId> · mcp:<userId> · channel:<participantId> ·
+  // function:<toolName> · trigger:<id> · system:<job>
+  actor: string;
+  // Apply each table's assistant access. Set for channel bots only.
+  restricted?: boolean;
+  // Customer-scoped tables: fill and filter `participant_id` with this.
+  participantId?: string;
+  // Datetimes come back in this zone, with the offset.
+  timeZone?: string;
+  limits?: DataCallerLimits;
+}
+
+export interface DataCallerLimits {
+  // Writes that add data stop at this size (the plan's ceiling).
+  maxBytes?: number | null;
+  maxTables?: number | null;
+  // The organization left the paid plan: reads and exports only.
+  readOnly?: boolean;
+}
+
+export interface DataUsage {
+  rowsRead: number;
+  rowsWritten: number;
+}
+
+export const DATA_ERROR_CODES = [
+  'invalid',
+  'not_found',
+  'forbidden',
+  'conflict',
+  'constraint',
+  'guard',
+  'too_large',
+  'full',
+  'read_only',
+  'deleted',
+  'not_initialised',
+  'restore',
+  'internal'
+] as const;
+export type DataErrorCode = (typeof DATA_ERROR_CODES)[number];
+
+// The one error the database object returns. Its message is written for the
+// person or model that made the call, never raw SQLite text.
+export class DataError extends Error {
+  readonly code: DataErrorCode;
+  readonly details: Record<string, unknown> | undefined;
+
+  constructor(
+    message: string,
+    code: DataErrorCode = 'invalid',
+    details?: Record<string, unknown>
+  ) {
+    super(message);
+    this.name = 'DataError';
+    this.code = code;
+    this.details = details;
+  }
+
+  // What the API answers with; its error handler reads both.
+  get status(): 400 | 403 | 404 | 409 | 500 {
+    return DATA_ERROR_STATUS[this.code];
+  }
+
+  toBody(): Record<string, unknown> {
+    return { code: this.code, ...(this.details ? { details: this.details } : {}) };
+  }
+}
+
+const DATA_ERROR_STATUS: Record<DataErrorCode, 400 | 403 | 404 | 409 | 500> = {
+  invalid: 400,
+  not_found: 404,
+  forbidden: 403,
+  conflict: 409,
+  constraint: 409,
+  guard: 409,
+  too_large: 400,
+  full: 409,
+  read_only: 403,
+  deleted: 409,
+  not_initialised: 404,
+  restore: 400,
+  internal: 500
+};
+
+export const isDataError = (error: unknown): error is DataError =>
+  error instanceof DataError;
+
+// What every database object method returns. Errors travel as values, not
+// throws: an RPC exception keeps only its message, and the caller needs the
+// code and the rows read before it.
+export type DataResult<T> =
+  | { ok: true; value: T; usage: DataUsage }
+  | {
+      ok: false;
+      error: {
+        code: DataErrorCode;
+        message: string;
+        details?: Record<string, unknown>;
+      };
+      usage: DataUsage;
+    };
+
+// Limits. SQLite in a Durable Object allows 100 columns per table, 100 bound
+// parameters per statement and 100 KB per statement; the rest are ours.
+export const DATA_MAX_PARAMS = 100;
+export const DATA_MAX_STATEMENT_BYTES = 90_000;
+// 100 minus id, created_at, updated_at, version, created_by, updated_by and
+// participant_id.
+export const DATA_MAX_COLUMNS = 93;
+export const DATA_MAX_TABLES = 50;
+export const DATA_DEFAULT_PAGE = 20;
+export const DATA_MAX_PAGE = 500;
+export const DATA_MAX_RESULT_BYTES = 1024 * 1024;
+// Past this many rows, an aggregate or a sort on an unindexed column needs a
+// filter on an indexed one: the object serves one request at a time.
+export const DATA_LARGE_TABLE_ROWS = 200_000;
+export const DATA_MAX_WRITE_OPERATIONS = 50;
+export const DATA_MAX_INSERT_ROWS = 500;
+export const DATA_MAX_ROWS_PER_WRITE = 5_000;
+export const DATA_MAX_VALUE_BYTES = 1_000_000;
+export const DATA_MAX_DESCRIPTION = 300;
+export const DATA_MAX_LABEL = 100;
+export const DATA_MAX_ENUM_VALUES = 100;
+export const DATA_MAX_IN_VALUES = 50;
+export const DATA_MAX_SORT_KEYS = 3;
+export const DATA_MAX_AGGREGATES = 5;
+export const DATA_MAX_GROUP_BY = 3;
+export const DATA_MAX_GROUPS = 100;
+// A type change copies the table in one request up to this size. A larger
+// table is copied in the background, this many rows a step (tens of
+// milliseconds, so other requests are served in between), and writes that
+// would change its rows wait until it is done.
+export const DATA_REBUILD_MAX_ROWS = 100_000;
+export const DATA_COPY_STEP_ROWS = 5_000;
+// A delete preview follows cascades through up to this many rows; past it,
+// its counts are lower bounds.
+export const DATA_IMPACT_MAX_ROWS = 200_000;
+export const DATA_IMPACT_MAX_IDS = DATA_MAX_ROWS_PER_WRITE;
+export const DATA_HISTORY_DAYS = 90;
+
+// What deleting rows does elsewhere: one entry per relation column that points
+// at a row the delete removes, cascades followed all the way down. `depth` is
+// how many cascades away from the deleted rows the pointed-at table is.
+export interface DataDeleteImpact {
+  table: string;
+  rows: number;
+  effects: {
+    table: string;
+    column: string;
+    onDelete: DataOnDelete;
+    rows: number;
+    depth: number;
+  }[];
+  // A restrict would refuse the delete.
+  blocked: boolean;
+  // Past DATA_IMPACT_MAX_ROWS: every count is at least what it says.
+  truncated: boolean;
+}
+
+// A table copy running in the background, as callers see it.
+export interface DataTableCopy {
+  table: string;
+  // The column whose change or drop started it.
+  column: string;
+  kind: 'change' | 'drop';
+  rows: number;
+  copied: number;
+  startedAt: string;
+  actor: string;
+  status: 'running' | 'failed';
+  error: string | null;
+}
+
+// The table as the owner would write it again: what `ganju.data.json` and an
+// export's schema.json hold.
+export const toTableInput = (table: DataTable): DataTableInput => ({
+  name: table.name,
+  label: table.label,
+  description: table.description,
+  displayColumn: table.displayColumn,
+  access: table.access,
+  customerScoped: table.customerScoped,
+  history: table.history,
+  columns: table.columns.map(column => {
+    const out: DataColumnInput = { name: column.name, type: column.type };
+    if (column.label) out.label = column.label;
+    if (column.description) out.description = column.description;
+    if (column.required) out.required = true;
+    if (column.unique) out.unique = true;
+    if (column.filterable) out.filterable = true;
+    if (column.searchable) out.searchable = true;
+    if (column.personal) out.personal = true;
+    if (column.values) out.values = column.values;
+    if (column.table) out.table = column.table;
+    if (column.onDelete) out.onDelete = column.onDelete;
+    if (column.min !== null) out.min = column.min;
+    if (column.default !== undefined) out.default = column.default;
+    return out;
+  })
+});

package/src/index.ts

@@ -344,6 +344,8 @@ import { citesUrl, citedPages, selectCitedSources } from './citations';
 import {
   refreshOAuthToken,
   OAuthReauthRequiredError,
+  OAuthRefreshError,
+  refreshFailureReauthReason,
   buildReauthMetadata,
   clearReauthMetadata,
   isCredentialNeedingReauth
@@ -557,6 +559,8 @@ export const utils = {
   selectCitedSources,
   refreshOAuthToken,
   OAuthReauthRequiredError,
+  OAuthRefreshError,
+  refreshFailureReauthReason,
   buildReauthMetadata,
   clearReauthMetadata,
   isCredentialNeedingReauth,

package/src/oauth.ts

@@ -24,6 +24,21 @@ export class OAuthReauthRequiredError extends Error {
   }
 }
 
+// The token endpoint answered with an error that isn't a reauth code. `status`
+// tells a rejection (4xx) from an outage (5xx); a network failure never gets
+// here, it throws fetch's own error.
+export class OAuthRefreshError extends Error {
+  readonly status: number;
+  readonly code: string | undefined;
+
+  constructor(status: number, code: string | undefined, body: string) {
+    super(`oauth refresh failed (${status}): ${body}`);
+    this.name = 'OAuthRefreshError';
+    this.status = status;
+    this.code = code;
+  }
+}
+
 const parseErrorCode = (body: string): string | undefined => {
   try {
     return (JSON.parse(body) as { error?: string })?.error;
@@ -52,7 +67,7 @@ export const refreshOAuthToken = async (
     if (code && constants.REAUTH_ERROR_CODES.includes(code)) {
       throw new OAuthReauthRequiredError(code);
     }
-    throw new Error(`oauth refresh failed (${response.status}): ${body}`);
+    throw new OAuthRefreshError(response.status, code, body);
   }
 
   const tokens = (await response.json()) as {
@@ -95,6 +110,32 @@ export const clearReauthMetadata = (
   return Object.keys(next).length > 0 ? next : null;
 };
 
+/**
+ * Why a failed refresh should flag the credential for a reconnect, or null
+ * when it shouldn't. A reauth code flags it at once. Any other rejection
+ * (4xx) flags it only once the access token has been expired for
+ * CREDENTIAL_REFRESH_GIVE_UP_MS, so a one-off doesn't. Never flagged: an
+ * outage (5xx), a network error, rate limiting (429), and errors about our
+ * own client id / secret, which no reconnect can fix.
+ */
+export const refreshFailureReauthReason = (
+  err: unknown,
+  expiresAt: Date | null,
+  now = Date.now()
+): string | null => {
+  if (err instanceof OAuthReauthRequiredError) return err.code;
+  if (!(err instanceof OAuthRefreshError)) return null;
+  if (err.status < 400 || err.status >= 500 || err.status === 429) return null;
+  if (err.code && constants.OAUTH_CLIENT_ERROR_CODES.includes(err.code)) {
+    return null;
+  }
+  if (!expiresAt) return null;
+  if (now - expiresAt.getTime() < constants.CREDENTIAL_REFRESH_GIVE_UP_MS) {
+    return null;
+  }
+  return err.code ? `refresh_rejected:${err.code}` : 'refresh_rejected';
+};
+
 export const isCredentialNeedingReauth = (metadata: unknown): boolean => {
   if (!metadata || typeof metadata !== 'object') return false;
   return (metadata as Record<string, unknown>).needsReauth === true;

package/src/resourceRerank.ts

@@ -15,7 +15,8 @@ type RerankResponse = {
 };
 
 /**
- * The cross-encoder second stage for resource search, or undefined when the
+ * The cross-encoder second stage for resource search, or undefined when it is
+ * off (RESOURCE_SEARCH_RERANK_ENABLED, unless `enabled` overrides it) or the
  * Worker has no `AI` binding — local dev without Workers AI keeps plain fusion.
  *
  * Returns one score per document, in input order. A response that doesn't
@@ -23,10 +24,12 @@ type RerankResponse = {
  * ranking would silently drop the chunks it left out.
  */
 export const createResourceReranker = (
-  source: EnvSource
+  source: EnvSource,
+  enabled: boolean = constants.RESOURCE_SEARCH_RERANK_ENABLED
 ):
   | ((query: string, documents: string[]) => Promise<number[] | null>)
   | undefined => {
+  if (!enabled) return undefined;
   const ai = (source.env as { AI?: RerankAiBinding } | undefined)?.AI;
   if (!ai) return undefined;