@ganju/utils 0.0.20 → 0.0.22

package/src/constants.ts

@@ -2197,6 +2197,36 @@ const DATA_PURGE_BATCH = 20;
 // Postgres write per query.
 const DATA_ACTIVITY_THROTTLE_MS = 60 * 60 * 1000;
 
+// Index advice. A column is worth an index when, over a week, at least 50
+// queries filtered or sorted on it and they read on average 1,000 rows and
+// 20 rows for every row returned. An index no query used in 30 days on a
+// table written at least 1,000 rows (and more rows written than queries) is
+// flagged as unused. Never dropped automatically.
+const DATA_INDEX_ADVICE_WINDOW_DAYS = 7;
+const DATA_INDEX_ADVICE_MIN_QUERIES = 50;
+const DATA_INDEX_ADVICE_MIN_ROWS_READ = 1_000;
+const DATA_INDEX_ADVICE_MIN_READ_RATIO = 20;
+const DATA_INDEX_UNUSED_WINDOW_DAYS = 30;
+const DATA_INDEX_UNUSED_MIN_ROWS_WRITTEN = 1_000;
+// Daily stats are kept past the longest window, then removed.
+const DATA_QUERY_STAT_KEEP_DAYS = 35;
+const DATA_INDEX_ADVICE_ADD = 'add' as 'add';
+const DATA_INDEX_ADVICE_UNUSED = 'unused' as 'unused';
+const DATA_INDEX_ADVICE_OPEN = 'open' as 'open';
+const DATA_INDEX_ADVICE_DISMISSED = 'dismissed' as 'dismissed';
+
+// The daily jobs (index advice and the abandoned-database flow) run on the
+// hourly cron at this UTC hour.
+const DATA_DAILY_JOB_HOUR_UTC = 13;
+
+// Days on Free before a database is warned about, reminded about, and listed
+// for Ganju to delete by hand.
+const DATA_ABANDON_WARN_DAYS = 30;
+const DATA_ABANDON_REMIND_DAYS = 41;
+const DATA_ABANDON_DELETE_DAYS = 44;
+// How long the export taken before a database is deleted is kept.
+const DATA_ABANDON_EXPORT_KEEP_DAYS = 90;
+
 interface PlanLimits {
   // null on any field means "no hard limit" for that plan.
   maxProjects: number | null;
@@ -2588,6 +2618,22 @@ export const constants = {
   DATA_DELETE_GRACE_DAYS,
   DATA_PURGE_BATCH,
   DATA_ACTIVITY_THROTTLE_MS,
+  DATA_INDEX_ADVICE_WINDOW_DAYS,
+  DATA_INDEX_ADVICE_MIN_QUERIES,
+  DATA_INDEX_ADVICE_MIN_ROWS_READ,
+  DATA_INDEX_ADVICE_MIN_READ_RATIO,
+  DATA_INDEX_UNUSED_WINDOW_DAYS,
+  DATA_INDEX_UNUSED_MIN_ROWS_WRITTEN,
+  DATA_QUERY_STAT_KEEP_DAYS,
+  DATA_INDEX_ADVICE_ADD,
+  DATA_INDEX_ADVICE_UNUSED,
+  DATA_INDEX_ADVICE_OPEN,
+  DATA_INDEX_ADVICE_DISMISSED,
+  DATA_DAILY_JOB_HOUR_UTC,
+  DATA_ABANDON_WARN_DAYS,
+  DATA_ABANDON_REMIND_DAYS,
+  DATA_ABANDON_DELETE_DAYS,
+  DATA_ABANDON_EXPORT_KEEP_DAYS,
   ARTIFACT_DATABASE_STATUS_CREATING,
   ARTIFACT_DATABASE_STATUS_READY,
   ARTIFACT_DATABASE_STATUS_FAILED,

package/src/data.ts

@@ -26,6 +26,8 @@ export {
   encodeCursor,
   decodeCursor,
   isSensitiveWrite,
+  fileUriOf,
+  unknownFileUris,
   tidyArgs,
   zoned
 } from './dataCompiler';
@@ -47,6 +49,8 @@ export {
   rowHistory,
   pruneHistory,
   deleteImpact,
+  fileLinks,
+  replaceFileLinks,
   createTable,
   updateTable,
   dropTable,
@@ -61,9 +65,18 @@ export {
   writeReachesCopy,
   assertActor,
   assertWritable,
+  assertNotSynced,
+  syncRows,
+  recordSyncFailure,
   exportRows
 } from './dataEngine';
-export type { SqlRow, SqlRunner } from './dataEngine';
+export type { SqlRow, SqlRunner, SheetSyncArgs } from './dataEngine';
+export { adviseIndexes, adviceDoubled } from './dataIndexAdvice';
+export type {
+  IndexAdviceCandidate,
+  IndexAdviceInput,
+  IndexAdviceStat
+} from './dataIndexAdvice';
 export {
   DATA_QUERY_PARAM_TYPES,
   DATA_MAX_QUERIES,
@@ -108,6 +121,7 @@ export {
 export type { DataToolDefinition, DataAdminAction } from './dataTools';
 export {
   DATA_TRIGGER_EVENTS,
+  DATA_CHANGE_EVENTS,
   DATA_TRIGGER_ACTIONS,
   DATA_MAX_TRIGGERS,
   DATA_TRIGGER_MAX_DEPTH,
@@ -115,10 +129,19 @@ export {
   DATA_TRIGGER_MAX_RATE,
   DATA_OUTBOX_BATCH,
   DATA_TRIGGER_MAX_MESSAGE_BYTES,
+  DATA_TIME_SCAN_MS,
+  DATA_TIME_MIN_MINUTES,
+  DATA_TIME_MAX_MINUTES,
+  DATA_MAX_TIME_TRIGGERS,
+  DATA_TIME_EVENTS_PER_SCAN,
+  DATA_TIME_EVENTS_PER_DAY,
   utf8Bytes,
   conditionMatches,
   renderTriggerText,
+  renderCustomerMessage,
+  readEventRow,
   normaliseTrigger,
+  normaliseCustomerAction,
   listTriggers,
   getTrigger,
   saveTrigger,
@@ -131,18 +154,25 @@ export {
   outboxSize,
   clearOutbox,
   triggersUsingColumn,
-  renameColumnInTriggers
+  renameColumnInTriggers,
+  timeTriggersCountingFrom,
+  syncTimeIndexes,
+  hasTimeTriggers,
+  scanTimeTriggers
 } from './dataTriggers';
 export type {
   DataTrigger,
   DataTriggerAction,
   DataTriggerActionType,
   DataTriggerCondition,
+  DataTriggerCustomerMessage,
   DataTriggerEvent,
   DataTriggerEventMessage,
   DataTriggerInput,
+  DataTriggerSchedule,
   DataTriggerTemplate,
   DataTriggerTest,
   DataTriggerWithSecret,
-  SavedTrigger
+  SavedTrigger,
+  TimeScanResult
 } from './dataTriggers';

package/src/dataApi.ts

@@ -27,7 +27,7 @@ export interface DataApiRoute {
   query?: z.ZodObject;
   // What a 200/201 answers with, in a sentence.
   returns: string;
-  status?: 200 | 201;
+  status?: 200 | 201 | 202;
   // Answers with a file instead of JSON.
   file?: boolean;
 }
@@ -59,6 +59,7 @@ const {
   DATA_TRIGGER_INPUT,
   DATA_TRIGGER_PATCH,
   DATA_TRIGGER_DEPLOY_BODY,
+  DATA_TRIGGER_TEST_SEND_BODY,
   DATA_TRIGGER_RUN_LIST_QUERY
 } = Schema;
 
@@ -69,7 +70,7 @@ export const DATA_API_ROUTES = {
     tag: 'Database',
     summary: "The project's database",
     description:
-      'Its status and size, and its tables when it is ready. `database` is null when the project has none.',
+      'Its status and size, and its tables when it is ready, with `indexAdvice`: the open index suggestions. `database` is null when the project has none.',
     returns: '{ database }'
   },
   create: {
@@ -139,6 +140,24 @@ export const DATA_API_ROUTES = {
       'A column change on a table over 100,000 rows copies it in the background; table changes, and writes that would change its rows, wait until it finishes (`copy` on `GET` shows how far it got). A copy that failed left the table as it was and stays in `copy` with its `error` until dismissed, or until the next column change.',
     returns: '{ copy: null }'
   },
+  applyIndexAdvice: {
+    method: 'post',
+    path: '/advice/:adviceId',
+    tag: 'Tables',
+    summary: 'Add a suggested index',
+    description:
+      "`GET` lists the open suggestions in `indexAdvice`: a column filtered or sorted on by 50 or more queries in a week that each read 1,000 or more rows, 20 for every row returned. This marks the column filterable, which indexes it, and records it in the change log. A suggestion of kind `unused` is only a flag; it can't be applied here, only dismissed.",
+    returns: '{ table }'
+  },
+  dismissIndexAdvice: {
+    method: 'delete',
+    path: '/advice/:adviceId',
+    tag: 'Tables',
+    summary: 'Dismiss an index suggestion',
+    description:
+      'It stays dismissed until its numbers double (queries, rows read per query, or rows written for an unused index).',
+    returns: '{ dismissed: true }'
+  },
   listTables: {
     method: 'get',
     path: '/table',
@@ -160,9 +179,21 @@ export const DATA_API_ROUTES = {
     path: '/table/:table',
     tag: 'Tables',
     summary: 'Rename a table or change its settings',
+    description:
+      "`source` (`{ uri, sheet?, key }`) keeps the table's rows in step with a spreadsheet or CSV in the project's knowledge, matched by `key`, a unique column: the sheet wins, and row writes to the table are refused until `source` is set back to null. It syncs when the file changes and from `POST …/sync`; `sync` on the table says how the last one went.",
     body: DATA_TABLE_PATCH,
     returns: '{ table }'
   },
+  syncTable: {
+    method: 'post',
+    path: '/table/:table/sync',
+    tag: 'Tables',
+    summary: 'Sync a table from its sheet now',
+    description:
+      "Queues a sync of a table that has a `source`: a Google Drive or OneDrive file is fetched again first if it changed. New keys are inserted, rows that differ updated (actor `system:sheet-sync`, so triggers fire), rows whose key is gone deleted. A sheet that doesn't fit (a missing key column, a repeated key, a value a column won't take) changes nothing; the table's `sync` says why.",
+    returns: '{ queued: true }',
+    status: 202
+  },
   dropTable: {
     method: 'delete',
     path: '/table/:table',
@@ -207,7 +238,7 @@ export const DATA_API_ROUTES = {
     tag: 'Rows',
     summary: 'List, filter and search rows',
     description:
-      'Up to 500 rows or 1 MB a page; `nextCursor` reads the next. `where` is JSON, e.g. `{"status":"open","price":{"lt":30000}}`. `sort` is a list of columns, `-` for descending. `q` searches the searchable columns.',
+      'Up to 500 rows or 1 MB a page; `nextCursor` reads the next. `where` is JSON, e.g. `{"status":"open","price":{"lt":30000}}`. `sort` is a list of columns, `-` for descending. `q` searches the searchable columns. A relation comes back as `{ id, label }`; a file column as `{ uri, id, title, mimeType }`, or `{ uri, missing: true }` when no resource has the uri any more. Writes take a file column\'s uri.',
     query: DATA_ROW_LIST_QUERY,
     returns: '{ table, rows, total?, nextCursor? }'
   },
@@ -328,7 +359,7 @@ export const DATA_API_ROUTES = {
     tag: 'Triggers',
     summary: 'Save a trigger',
     description:
-      'Runs one action when rows of `table` are inserted, updated or deleted, and `when` holds: `{ "status": { "changedTo": "READY" } }` fires on the transition only; `changedFrom`, `changed: true` and `equals` test a column too, and `all` / `any` combine them. The action is one of `function` (a function of this project, given `{ event, table, row, previous, actor, eventId, triggerId }`), `webhook` (an https URL, sent a POST signed in `x-ganju-signature`), `chat` (`{ channelId, chatId, text }`), `email` (`{ to, subject, text }`) or `customer` (`{ text, template? }`, to the row\'s customer on a customer-scoped table). Texts take `{{field}}` from the row, inserted as plain text. Saving under an existing name replaces it, unless `replace` is false; with no `name`, it always creates a new trigger, named from the table, event and action and numbered past those taken. Left out, `enabled` keeps a saved trigger on or off as it was. A new webhook answers with its signing `secret`, shown this once. An `email` goes to the organization\'s members; any other address is sent a link to confirm first, and those still waiting come back as `confirming`. Delivery is at least once: every event carries an `eventId`.',
+      'Runs one action when rows of `table` are inserted, updated or deleted, and `when` holds: `{ "status": { "changedTo": "READY" } }` fires on the transition only; `changedFrom`, `changed: true` and `equals` test a column too, and `all` / `any` combine them. With `"on": "time"` it fires instead once a row comes due, with nothing written: `after` is `{ column, minutes }`, the column `created_at`, `updated_at` or a date or datetime column (a date counts from midnight in the database\'s time zone) and `minutes` 5 to 43,200, negative on a date or datetime for before it. Its `when` tests `equals` only, read when the row comes due; rows already due when it is saved don\'t fire, and each row fires once per due time (an `updated_at` trigger again after the next change). Due rows are looked for every 5 minutes, up to 100 events per trigger per scan; a database holds up to 10 time triggers, and the database\'s time triggers share 2,000 events a day (UTC): past that, each time trigger with another row due that day is turned off and the organization\'s admins are emailed. A time trigger is also turned off when its runs are paused (the plan no longer includes Data, or the month\'s trigger runs reached the plan\'s limit). The action is one of `function` (a function of this project, given `{ event, table, row, previous, actor, eventId, triggerId, dueAt? }`), `webhook` (an https URL, sent a POST signed in `x-ganju-signature`), `chat` (`{ channelId, chatId, text }`), `email` (`{ to, subject, text }`) or `customer` (`{ text, template? }`, to the row\'s customer on a customer-scoped table). Texts take `{{field}}` from the row, inserted as plain text. Saving under an existing name replaces it, unless `replace` is false; with no `name`, it always creates a new trigger, named from the table, event and action and numbered past those taken. Left out, `enabled` keeps a saved trigger on or off as it was. A new webhook answers with its signing `secret`, shown this once. An `email` goes to the organization\'s members; any other address is sent a link to confirm first, and those still waiting come back as `confirming`. Delivery is at least once: every event carries an `eventId`.',
     body: DATA_TRIGGER_INPUT,
     returns: '{ trigger, secret?, confirming? }',
     status: 201
@@ -360,6 +391,25 @@ export const DATA_API_ROUTES = {
     body: DATA_TRIGGER_DEPLOY_BODY,
     returns: '{ triggers, removed, secrets, confirming? }'
   },
+  listTestRecipients: {
+    method: 'get',
+    path: '/trigger/test-send',
+    tag: 'Triggers',
+    summary: 'Where a test send can go',
+    description:
+      "The caller's own chats with this project's bots: the ones they linked to their Ganju account by sending `/link` to the bot. Empty when there are none.",
+    returns: '{ recipients }'
+  },
+  testSend: {
+    method: 'post',
+    path: '/trigger/test-send',
+    tag: 'Triggers',
+    summary: 'Send a test of a customer message',
+    description:
+      "Sends a `customer` action, saved or not, to the caller's own chat with the bot instead of the row's customer: built from the row `rowId` (the table's newest row by default), the same as a real run. `participantId` picks one of `GET /trigger/test-send`'s recipients when there are several. With `asTemplate`, a WhatsApp template goes out even inside the 24-hour window. Not a run: it isn't listed with any trigger's runs and doesn't count toward the daily limit per customer. Five a minute per user. A channel's refusal (the template not approved, outside the window with no template, the daily limit) comes back as the error.",
+    body: DATA_TRIGGER_TEST_SEND_BODY,
+    returns: '{ sent, text, template, rowId, recipient }'
+  },
   listTriggerRuns: {
     method: 'get',
     path: '/trigger/:name/run',

package/src/dataAttachment.ts

@@ -0,0 +1,23 @@
+import { Schema } from './schema';
+
+export interface DataAttachment {
+  table: string;
+  column: string;
+  index: boolean;
+}
+
+// The Data upload a knowledge item came from, read from its metadata, or
+// null for any other item.
+export const dataAttachmentOf = (metadata: unknown): DataAttachment | null => {
+  if (!metadata || typeof metadata !== 'object') return null;
+  const parsed = Schema.DATA_ATTACHMENT_METADATA.safeParse(
+    (metadata as { dataAttachment?: unknown }).dataAttachment
+  );
+  return parsed.success ? parsed.data : null;
+};
+
+// Uploads from Data are stored, not indexed, unless their column asks.
+export const skipsIndexing = (metadata: unknown): boolean => {
+  const attachment = dataAttachmentOf(metadata);
+  return attachment !== null && !attachment.index;
+};

package/src/dataCompiler.ts

@@ -8,6 +8,7 @@ import {
   DATA_ACCESS_RANK,
   DATA_MAX_IN_VALUES,
   DATA_MAX_PARAMS,
+  DATA_FILE_URI_MAX,
   DATA_MAX_VALUE_BYTES,
   DataError
 } from './dataTypes';
@@ -193,7 +194,10 @@ const LOCAL_DATETIME =
 // "2026-10-21T15:00:00" with no offset, read as a wall-clock time in
 // `timeZone` (UTC without one). `new Date()` would read it in the server's
 // zone: UTC on Workers, the developer's own zone on a laptop.
-const localDatetime = (value: string, timeZone?: string): Date | null => {
+export const localDatetime = (
+  value: string,
+  timeZone?: string
+): Date | null => {
   const m = LOCAL_DATETIME.exec(value.trim());
   if (!m) return null;
   const [year, month, day, hour, minute, second] = m
@@ -260,6 +264,37 @@ const checkSize = (label: string, value: string) => {
   return value;
 };
 
+// A knowledge item's uri, given as the text or as { uri } (how reads return
+// it). Any scheme: knowledge holds resource://, website and drive uris.
+export const fileUriOf = (value: unknown, label: string): string => {
+  const raw =
+    value !== null && typeof value === 'object' && !Array.isArray(value)
+      ? (value as { uri?: unknown }).uri
+      : value;
+  const uri = typeof raw === 'string' ? raw.trim() : '';
+  if (!/^[a-z][a-z0-9+.-]*:\/\/\S+$/i.test(uri)) {
+    throw new DataError(
+      `${label} takes a knowledge item's uri, e.g. resource://menu-photo.`
+    );
+  }
+  if (uri.length > DATA_FILE_URI_MAX) {
+    throw new DataError(
+      `${label}: a uri is at most ${DATA_FILE_URI_MAX} characters.`,
+      'too_large'
+    );
+  }
+  return uri;
+};
+
+// A write that links uris no item of the project has: the workers check,
+// since the database object can't see knowledge.
+export const unknownFileUris = (uris: string[]) =>
+  new DataError(
+    `No resource in this project has the uri ${uris.join(', ')}; link an existing resource by its uri.`,
+    'invalid',
+    { uris }
+  );
+
 // Turns a value from a caller into what the column stores, or explains why
 // it can't. `timeZone` is the project's, for datetimes sent without an
 // offset.
@@ -337,6 +372,8 @@ export const toStored = (
         throw new DataError(`${label} must be a URL starting with http(s)://.`);
       }
       return checkSize(label, value.trim());
+    case 'file':
+      return fileUriOf(value, label);
     case 'phone': {
       if (typeof value !== 'string' && typeof value !== 'number') {
         throw new DataError(`${label} must be a phone number.`);
@@ -429,6 +466,8 @@ export interface CompileContext {
   table: DataTable;
   joins: Map<string, SqlJoin>;
   params: SqlValue[];
+  // The queried table's own columns a `where` filters on, for index advice.
+  filtered?: Set<string>;
 }
 
 // "column" or "relation.column", adding the join a relation path needs.
@@ -548,6 +587,7 @@ export const compileWhere = (
       continue;
     }
     const resolved = resolvePath(ctx, key);
+    if (resolved.own) ctx.filtered?.add(resolved.column.name);
     // { "customer": { "name": "Ana" } } reads as "customer.name": models
     // write relation filters this way unprompted.
     if (

package/src/dataDdl.ts

@@ -3,6 +3,7 @@
 // ours, and every owner-supplied value is a literal escaped by sqlLiteral.
 import { changeCaptureTriggers } from './dataTriggers';
 import { quoteIdentifier as q } from './dataIdentifiers';
+import { DATA_FILE_URI_MAX } from './dataTypes';
 import type {
   DataColumn,
   DataColumnType,
@@ -412,6 +413,12 @@ export const invalidAfterConversionSql = (
       `NOT ((${converted}) LIKE 'http://%' OR (${converted}) LIKE 'https://%')`
     );
   }
+  // A file column holds a knowledge item's uri: scheme://…, no spaces.
+  if (to.type === 'file') {
+    parts.push(
+      `NOT ((${converted}) GLOB '[A-Za-z]*://?*' AND (${converted}) NOT GLOB '*[ ' || char(9) || char(10) || char(13) || ']*' AND length(${converted}) <= ${DATA_FILE_URI_MAX})`
+    );
+  }
   if (to.type === 'relation' && to.table) {
     parts.push(
       `((${converted}) IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${q(to.table)} WHERE id = (${converted})))`