@ganju/utils 0.0.18 → 0.0.21

package/src/dataApi.ts

@@ -59,6 +59,7 @@ const {
   DATA_TRIGGER_INPUT,
   DATA_TRIGGER_PATCH,
   DATA_TRIGGER_DEPLOY_BODY,
+  DATA_TRIGGER_TEST_SEND_BODY,
   DATA_TRIGGER_RUN_LIST_QUERY
 } = Schema;
 
@@ -69,7 +70,7 @@ export const DATA_API_ROUTES = {
     tag: 'Database',
     summary: "The project's database",
     description:
-      'Its status and size, and its tables when it is ready. `database` is null when the project has none.',
+      'Its status and size, and its tables when it is ready, with `indexAdvice`: the open index suggestions. `database` is null when the project has none.',
     returns: '{ database }'
   },
   create: {
@@ -139,6 +140,24 @@ export const DATA_API_ROUTES = {
       'A column change on a table over 100,000 rows copies it in the background; table changes, and writes that would change its rows, wait until it finishes (`copy` on `GET` shows how far it got). A copy that failed left the table as it was and stays in `copy` with its `error` until dismissed, or until the next column change.',
     returns: '{ copy: null }'
   },
+  applyIndexAdvice: {
+    method: 'post',
+    path: '/advice/:adviceId',
+    tag: 'Tables',
+    summary: 'Add a suggested index',
+    description:
+      "`GET` lists the open suggestions in `indexAdvice`: a column filtered or sorted on by 50 or more queries in a week that each read 1,000 or more rows, 20 for every row returned. This marks the column filterable, which indexes it, and records it in the change log. A suggestion of kind `unused` is only a flag; it can't be applied here, only dismissed.",
+    returns: '{ table }'
+  },
+  dismissIndexAdvice: {
+    method: 'delete',
+    path: '/advice/:adviceId',
+    tag: 'Tables',
+    summary: 'Dismiss an index suggestion',
+    description:
+      'It stays dismissed until its numbers double (queries, rows read per query, or rows written for an unused index).',
+    returns: '{ dismissed: true }'
+  },
   listTables: {
     method: 'get',
     path: '/table',
@@ -207,7 +226,7 @@ export const DATA_API_ROUTES = {
     tag: 'Rows',
     summary: 'List, filter and search rows',
     description:
-      'Up to 500 rows or 1 MB a page; `nextCursor` reads the next. `where` is JSON, e.g. `{"status":"open","price":{"lt":30000}}`. `sort` is a list of columns, `-` for descending. `q` searches the searchable columns.',
+      'Up to 500 rows or 1 MB a page; `nextCursor` reads the next. `where` is JSON, e.g. `{"status":"open","price":{"lt":30000}}`. `sort` is a list of columns, `-` for descending. `q` searches the searchable columns. A relation comes back as `{ id, label }`; a file column as `{ uri, id, title, mimeType }`, or `{ uri, missing: true }` when no resource has the uri any more. Writes take a file column\'s uri.',
     query: DATA_ROW_LIST_QUERY,
     returns: '{ table, rows, total?, nextCursor? }'
   },
@@ -328,9 +347,9 @@ export const DATA_API_ROUTES = {
     tag: 'Triggers',
     summary: 'Save a trigger',
     description:
-      'Runs one action when rows of `table` are inserted, updated or deleted, and `when` holds: `{ "status": { "changedTo": "READY" } }` fires on the transition only; `changedFrom`, `changed: true` and `equals` test a column too, and `all` / `any` combine them. The action is one of `function` (a function of this project, given `{ event, table, row, previous, actor, eventId, triggerId }`), `webhook` (an https URL, sent a POST signed in `x-ganju-signature`), `chat` (`{ channelId, chatId, text }`), `email` (`{ to, subject, text }`) or `customer` (`{ text, template? }`, to the row\'s customer on a customer-scoped table). Texts take `{{field}}` from the row, inserted as plain text. Saving under an existing name replaces it, unless `replace` is false; with no `name`, it always creates a new trigger, named from the table, event and action and numbered past those taken. Left out, `enabled` keeps a saved trigger on or off as it was. A new webhook answers with its signing `secret`, shown this once. Delivery is at least once: every event carries an `eventId`.',
+      'Runs one action when rows of `table` are inserted, updated or deleted, and `when` holds: `{ "status": { "changedTo": "READY" } }` fires on the transition only; `changedFrom`, `changed: true` and `equals` test a column too, and `all` / `any` combine them. The action is one of `function` (a function of this project, given `{ event, table, row, previous, actor, eventId, triggerId }`), `webhook` (an https URL, sent a POST signed in `x-ganju-signature`), `chat` (`{ channelId, chatId, text }`), `email` (`{ to, subject, text }`) or `customer` (`{ text, template? }`, to the row\'s customer on a customer-scoped table). Texts take `{{field}}` from the row, inserted as plain text. Saving under an existing name replaces it, unless `replace` is false; with no `name`, it always creates a new trigger, named from the table, event and action and numbered past those taken. Left out, `enabled` keeps a saved trigger on or off as it was. A new webhook answers with its signing `secret`, shown this once. An `email` goes to the organization\'s members; any other address is sent a link to confirm first, and those still waiting come back as `confirming`. Delivery is at least once: every event carries an `eventId`.',
     body: DATA_TRIGGER_INPUT,
-    returns: '{ trigger, secret? }',
+    returns: '{ trigger, secret?, confirming? }',
     status: 201
   },
   updateTrigger: {
@@ -356,9 +375,28 @@ export const DATA_API_ROUTES = {
     tag: 'Triggers',
     summary: 'Deploy the triggers of ganju.json',
     description:
-      "What `ganju deploy` sends: every trigger listed is saved as managed by the CLI, and the CLI's earlier ones not listed are deleted. Triggers made in the dashboard are left alone. All or nothing. New webhooks' secrets come back once, by name.",
+      "What `ganju deploy` sends: every trigger listed is saved as managed by the CLI, and the CLI's earlier ones not listed are deleted. Triggers made in the dashboard are left alone. All or nothing. New webhooks' secrets come back once, by name, and email addresses still to confirm as `confirming`.",
     body: DATA_TRIGGER_DEPLOY_BODY,
-    returns: '{ triggers, removed, secrets }'
+    returns: '{ triggers, removed, secrets, confirming? }'
+  },
+  listTestRecipients: {
+    method: 'get',
+    path: '/trigger/test-send',
+    tag: 'Triggers',
+    summary: 'Where a test send can go',
+    description:
+      "The caller's own chats with this project's bots: the ones they linked to their Ganju account by sending `/link` to the bot. Empty when there are none.",
+    returns: '{ recipients }'
+  },
+  testSend: {
+    method: 'post',
+    path: '/trigger/test-send',
+    tag: 'Triggers',
+    summary: 'Send a test of a customer message',
+    description:
+      "Sends a `customer` action, saved or not, to the caller's own chat with the bot instead of the row's customer: built from the row `rowId` (the table's newest row by default), the same as a real run. `participantId` picks one of `GET /trigger/test-send`'s recipients when there are several. With `asTemplate`, a WhatsApp template goes out even inside the 24-hour window. Not a run: it isn't listed with any trigger's runs and doesn't count toward the daily limit per customer. Five a minute per user. A channel's refusal (the template not approved, outside the window with no template, the daily limit) comes back as the error.",
+    body: DATA_TRIGGER_TEST_SEND_BODY,
+    returns: '{ sent, text, template, rowId, recipient }'
   },
   listTriggerRuns: {
     method: 'get',

package/src/dataAttachment.ts

@@ -0,0 +1,23 @@
+import { Schema } from './schema';
+
+export interface DataAttachment {
+  table: string;
+  column: string;
+  index: boolean;
+}
+
+// The Data upload a knowledge item came from, read from its metadata, or
+// null for any other item.
+export const dataAttachmentOf = (metadata: unknown): DataAttachment | null => {
+  if (!metadata || typeof metadata !== 'object') return null;
+  const parsed = Schema.DATA_ATTACHMENT_METADATA.safeParse(
+    (metadata as { dataAttachment?: unknown }).dataAttachment
+  );
+  return parsed.success ? parsed.data : null;
+};
+
+// Uploads from Data are stored, not indexed, unless their column asks.
+export const skipsIndexing = (metadata: unknown): boolean => {
+  const attachment = dataAttachmentOf(metadata);
+  return attachment !== null && !attachment.index;
+};

package/src/dataCompiler.ts

@@ -8,6 +8,7 @@ import {
   DATA_ACCESS_RANK,
   DATA_MAX_IN_VALUES,
   DATA_MAX_PARAMS,
+  DATA_FILE_URI_MAX,
   DATA_MAX_VALUE_BYTES,
   DataError
 } from './dataTypes';
@@ -260,6 +261,37 @@ const checkSize = (label: string, value: string) => {
   return value;
 };
 
+// A knowledge item's uri, given as the text or as { uri } (how reads return
+// it). Any scheme: knowledge holds resource://, website and drive uris.
+export const fileUriOf = (value: unknown, label: string): string => {
+  const raw =
+    value !== null && typeof value === 'object' && !Array.isArray(value)
+      ? (value as { uri?: unknown }).uri
+      : value;
+  const uri = typeof raw === 'string' ? raw.trim() : '';
+  if (!/^[a-z][a-z0-9+.-]*:\/\/\S+$/i.test(uri)) {
+    throw new DataError(
+      `${label} takes a knowledge item's uri, e.g. resource://menu-photo.`
+    );
+  }
+  if (uri.length > DATA_FILE_URI_MAX) {
+    throw new DataError(
+      `${label}: a uri is at most ${DATA_FILE_URI_MAX} characters.`,
+      'too_large'
+    );
+  }
+  return uri;
+};
+
+// A write that links uris no item of the project has: the workers check,
+// since the database object can't see knowledge.
+export const unknownFileUris = (uris: string[]) =>
+  new DataError(
+    `No resource in this project has the uri ${uris.join(', ')}; link an existing resource by its uri.`,
+    'invalid',
+    { uris }
+  );
+
 // Turns a value from a caller into what the column stores, or explains why
 // it can't. `timeZone` is the project's, for datetimes sent without an
 // offset.
@@ -337,6 +369,8 @@ export const toStored = (
         throw new DataError(`${label} must be a URL starting with http(s)://.`);
       }
       return checkSize(label, value.trim());
+    case 'file':
+      return fileUriOf(value, label);
     case 'phone': {
       if (typeof value !== 'string' && typeof value !== 'number') {
         throw new DataError(`${label} must be a phone number.`);
@@ -429,6 +463,8 @@ export interface CompileContext {
   table: DataTable;
   joins: Map<string, SqlJoin>;
   params: SqlValue[];
+  // The queried table's own columns a `where` filters on, for index advice.
+  filtered?: Set<string>;
 }
 
 // "column" or "relation.column", adding the join a relation path needs.
@@ -548,6 +584,7 @@ export const compileWhere = (
       continue;
     }
     const resolved = resolvePath(ctx, key);
+    if (resolved.own) ctx.filtered?.add(resolved.column.name);
     // { "customer": { "name": "Ana" } } reads as "customer.name": models
     // write relation filters this way unprompted.
     if (

package/src/dataDdl.ts

@@ -3,6 +3,7 @@
 // ours, and every owner-supplied value is a literal escaped by sqlLiteral.
 import { changeCaptureTriggers } from './dataTriggers';
 import { quoteIdentifier as q } from './dataIdentifiers';
+import { DATA_FILE_URI_MAX } from './dataTypes';
 import type {
   DataColumn,
   DataColumnType,
@@ -412,6 +413,12 @@ export const invalidAfterConversionSql = (
       `NOT ((${converted}) LIKE 'http://%' OR (${converted}) LIKE 'https://%')`
     );
   }
+  // A file column holds a knowledge item's uri: scheme://…, no spaces.
+  if (to.type === 'file') {
+    parts.push(
+      `NOT ((${converted}) GLOB '[A-Za-z]*://?*' AND (${converted}) NOT GLOB '*[ ' || char(9) || char(10) || char(13) || ']*' AND length(${converted}) <= ${DATA_FILE_URI_MAX})`
+    );
+  }
   if (to.type === 'relation' && to.table) {
     parts.push(
       `((${converted}) IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${q(to.table)} WHERE id = (${converted})))`

package/src/dataEngine.ts

@@ -30,6 +30,7 @@ import {
   compileWhere,
   decodeCursor,
   encodeCursor,
+  fileUriOf,
   fromStored,
   ftsMatch,
   isScoped,
@@ -74,6 +75,7 @@ import {
   DATA_COPY_STEP_ROWS,
   DATA_IMPACT_MAX_IDS,
   DATA_IMPACT_MAX_ROWS,
+  DATA_FILE_LINKS_MAX_URIS,
   DATA_REBUILD_MAX_ROWS,
   DataError
 } from './dataTypes';
@@ -85,11 +87,17 @@ import type {
   DataColumnType,
   DataDefaultValue,
   DataDeleteImpact,
+  DataFileLinkChange,
+  DataFileLinks,
+  DataFileLinksReplaced,
   DataOnDelete,
+  DataQueryShape,
   DataSchema,
   DataTable,
   DataTableCopy,
-  DataTableInput
+  DataTableInput,
+  DataUsage,
+  DataWriteShape
 } from './dataTypes';
 
 export type SqlRow = Record<string, SqlValue>;
@@ -101,6 +109,11 @@ export interface SqlRunner {
   iterate(sql: string, params?: readonly SqlValue[]): Iterable<SqlRow>;
   transaction<T>(fn: () => T): T;
   databaseSize(): number;
+  // Rows read and written so far in this call, and where a query or write
+  // notes what it did. Optional: tests and internal runners skip them.
+  usageSoFar?(): DataUsage;
+  noteQuery?(shape: DataQueryShape): void;
+  noteWrite?(shape: DataWriteShape): void;
 }
 
 const first = (runner: SqlRunner, sql: string, params?: readonly SqlValue[]) =>
@@ -483,9 +496,14 @@ export const normaliseColumn = (
   if (column.unique && !UNIQUE_TYPES.has(column.type)) {
     throw new DataError(`${label}: a ${column.type} column can't be unique.`);
   }
-  if (column.searchable && !SEARCHABLE_TYPES.has(column.type)) {
+  // On a file column, searchable means its uploads are indexed in knowledge.
+  if (
+    column.searchable &&
+    !SEARCHABLE_TYPES.has(column.type) &&
+    column.type !== 'file'
+  ) {
     throw new DataError(
-      `${label}: only text columns can be searchable (text, long_text, email, url, phone, enum).`
+      `${label}: only text and file columns can be searchable (text, long_text, email, url, phone, enum, file).`
     );
   }
   // The search table has a column of its own called rank.
@@ -786,6 +804,8 @@ const shapeRow = (
                   ? fromStored(display, label, timeZone)
                   : label
             };
+    } else if (column.type === 'file') {
+      row[column.name] = value === null ? null : { uri: String(value) };
     } else {
       row[column.name] = value;
     }
@@ -890,8 +910,20 @@ export const query = (
     caller,
     table,
     joins: new Map(),
-    params: []
+    params: [],
+    filtered: new Set()
   };
+  // What this query read, noted for index advice once it has its rows.
+  const readBefore = runner.usageSoFar?.().rowsRead ?? 0;
+  const sorted: string[] = [];
+  const noteShape = (returned: number) =>
+    runner.noteQuery?.({
+      table: table.name,
+      filtered: [...(ctx.filtered ?? [])],
+      sorted,
+      rowsRead: (runner.usageSoFar?.().rowsRead ?? 0) - readBefore,
+      returned
+    });
   const indexed = new Set<string>();
   const where = compileWhere(ctx, args.where, indexed);
   if (isScoped(caller, table)) {
@@ -907,7 +939,10 @@ export const query = (
   }
 
   if (args.aggregate !== undefined && args.aggregate !== null) {
-    return aggregate(runner, ctx, args, where, searching, indexed.size > 0);
+    const result = aggregate(runner, ctx, args, where, searching, indexed.size > 0);
+    const groups = (result as { groups?: unknown }).groups;
+    noteShape(Array.isArray(groups) ? groups.length : 1);
+    return result;
   }
 
   const limit = args.limit === undefined || args.limit === null ? DATA_DEFAULT_PAGE : args.limit;
@@ -946,6 +981,7 @@ export const query = (
       descending,
       ...(servedByIndex ? {} : { unindexed: resolved.column.name })
     });
+    if (resolved.own && resolved.column.name !== 'id') sorted.push(resolved.column.name);
     if (resolved.own && resolved.column.name === 'id') {
       endsUnique = true;
       break;
@@ -1043,6 +1079,7 @@ export const query = (
   }
 
   const { rows: out, last, more } = result;
+  noteShape(out.length);
   const includeTruncated =
     args.include !== undefined && args.include !== null
       ? includeRelations(
@@ -1055,11 +1092,13 @@ export const query = (
           DATA_MAX_RESULT_BYTES - result.bytes
         )
       : false;
+  const fileFields = fileFieldPaths(schema, table, args.include);
   return {
     table: table.name,
     ...(isScoped(caller, table) ? { scope: SCOPE_NOTE } : {}),
     ...(result.total !== undefined ? { total: result.total } : {}),
     ...(matched ? { searchMatched: matched } : {}),
+    ...(fileFields.length ? { fileFields } : {}),
     rows: out,
     ...(includeTruncated
       ? {
@@ -1079,6 +1118,29 @@ export const query = (
   };
 };
 
+// Where a query's rows hold file values, as key paths within each row: the
+// table's file columns, and those of each included relation. The workers
+// fill those in from knowledge and drop this before answering; a json
+// column holding { "uri": … } is never mistaken for one.
+const fileFieldPaths = (
+  schema: DataSchema,
+  table: DataTable,
+  include: unknown
+): string[][] => {
+  const own = table.columns
+    .filter(column => column.type === 'file')
+    .map(column => [column.name]);
+  const included = (Array.isArray(include) ? include : []).flatMap(name => {
+    const column = typeof name === 'string' ? columnFor(table, name) : undefined;
+    if (!column || isSystemColumn(column) || column.type !== 'relation') return [];
+    const target = schema.tables.find(t => t.name === column.table);
+    return (target?.columns ?? [])
+      .filter(c => c.type === 'file')
+      .map(c => [name as string, c.name]);
+  });
+  return [...own, ...included];
+};
+
 // Replaces each relation in `rows` named in `include` with the related row,
 // read as the caller may see it, within `budget` bytes. Returns whether any
 // was left as { id, label } to stay inside it.
@@ -1840,7 +1902,12 @@ export const write = (
     const results: WriteResult[] = [];
     operations.forEach((op, index) => {
       try {
+        const before = runner.usageSoFar?.().rowsWritten ?? 0;
         results.push(runOperation(runner, schema, caller, op, index, results));
+        runner.noteWrite?.({
+          table: results[results.length - 1].table,
+          rowsWritten: (runner.usageSoFar?.().rowsWritten ?? 0) - before
+        });
       } catch (error) {
         if (error instanceof DataError) {
           const label = isPlainObject(op) ? `${String(op.op)} ${String(op.table)}` : '?';
@@ -1979,6 +2046,129 @@ export const deleteImpact = (
   };
 };
 
+// File links
+
+const fileColumns = (schema: DataSchema) =>
+  schema.tables.flatMap(table =>
+    table.columns
+      .filter(column => column.type === 'file')
+      .map(column => ({ table, column }))
+  );
+
+const fileUriList = (value: unknown): string[] => {
+  if (
+    !Array.isArray(value) ||
+    value.length === 0 ||
+    value.length > DATA_FILE_LINKS_MAX_URIS
+  ) {
+    throw new DataError(
+      `"uris" lists 1 to ${DATA_FILE_LINKS_MAX_URIS} knowledge item uris.`
+    );
+  }
+  return [...new Set(value.map(uri => fileUriOf(uri, 'uris')))];
+};
+
+// The rows whose file columns point at `uris`, per column: knowledge asks
+// before it deletes an item. Each file column is indexed.
+export const fileLinks = (
+  runner: SqlRunner,
+  schema: DataSchema,
+  caller: DataCaller,
+  rawArgs: unknown
+): DataFileLinks => {
+  assertActor(caller);
+  if (caller.restricted) {
+    throw new DataError('File links are for the owner.', 'forbidden');
+  }
+  const uris = JSON.stringify(fileUriList(tidyArgs(rawArgs).uris));
+  const links: DataFileLinks['links'] = [];
+  for (const { table, column } of fileColumns(schema)) {
+    const rows = Number(
+      first(
+        runner,
+        `SELECT count(*) AS n FROM ${q(table.name)} WHERE ${q(column.name)} IN (SELECT value FROM json_each(?))`,
+        [uris]
+      )?.n ?? 0
+    );
+    if (rows === 0) continue;
+    links.push({
+      table: table.name,
+      tableLabel: table.label,
+      column: column.name,
+      columnLabel: column.label,
+      required: column.required,
+      rows
+    });
+  }
+  return { links };
+};
+
+// Points the links at `from` to `to`, or clears them (`to: null`) once the
+// item is gone. A required column keeps its link, which then reads as
+// missing. Recorded in row history; triggers don't see it.
+export const replaceFileLinks = (
+  runner: SqlRunner,
+  schema: DataSchema,
+  caller: DataCaller,
+  rawArgs: unknown
+): DataFileLinksReplaced => {
+  assertActor(caller);
+  if (caller.restricted) {
+    throw new DataError('File links are for the owner.', 'forbidden');
+  }
+  const raw = tidyArgs(rawArgs).changes;
+  if (
+    !Array.isArray(raw) ||
+    raw.length === 0 ||
+    raw.length > DATA_FILE_LINKS_MAX_URIS
+  ) {
+    throw new DataError(
+      `"changes" lists 1 to ${DATA_FILE_LINKS_MAX_URIS} { from, to } pairs.`
+    );
+  }
+  const changes: DataFileLinkChange[] = raw.map(change => {
+    if (!isPlainObject(change)) {
+      throw new DataError('Each change is { from, to }; to is null to clear.');
+    }
+    return {
+      from: fileUriOf(change.from, 'from'),
+      to: change.to === null ? null : fileUriOf(change.to, 'to')
+    };
+  });
+  const columns = fileColumns(schema);
+  if (columns.length === 0) return { updated: 0, kept: [] };
+  assertWritable(runner, caller, changes.some(change => change.to !== null));
+  return runner.transaction(() => {
+    setContextActor(runner, caller.actor);
+    runner.all('DELETE FROM _ganju_changes');
+    let updated = 0;
+    const kept: DataFileLinksReplaced['kept'] = [];
+    for (const { table, column } of columns) {
+      for (const change of changes) {
+        if (change.to === null && column.required) {
+          const rows = Number(
+            first(
+              runner,
+              `SELECT count(*) AS n FROM ${q(table.name)} WHERE ${q(column.name)} = ?`,
+              [change.from]
+            )?.n ?? 0
+          );
+          if (rows > 0) kept.push({ table: table.name, column: column.name, rows });
+          continue;
+        }
+        updated += runner.all(
+          `UPDATE ${q(table.name)} SET ${q(column.name)} = ?, version = version + 1, updated_at = ${DATA_NOW_SQL}, updated_by = ? WHERE ${q(column.name)} = ? RETURNING id`,
+          [change.to, caller.actor, change.from]
+        ).length;
+      }
+    }
+    // Tidying links after a knowledge change isn't the owner's edit: it
+    // stays in row history but fires no trigger.
+    runner.all('DELETE FROM _ganju_changes');
+    return { updated, kept };
+  });
+};
+
 // Row history
 
 export const rowHistory = (

package/src/dataIndexAdvice.ts

@@ -0,0 +1,127 @@
+// Which columns of a database are worth an index, and which indexes nobody
+// uses, from the API's daily query stats and the database's schema. Pure, so
+// the daily job and the tests apply the same rule.
+import { constants } from './constants';
+import { isIndexedColumn } from './dataDdl';
+import type { DataSchema } from './dataTypes';
+
+// A column's (or, with column '', a table's) sums over a window.
+export interface IndexAdviceStat {
+  tableName: string;
+  columnName: string;
+  queries: number;
+  rowsRead: number;
+  rowsReturned: number;
+  rowsWritten: number;
+}
+
+export interface IndexAdviceCandidate {
+  tableName: string;
+  columnName: string;
+  kind: 'add' | 'unused';
+  queries: number;
+  avgRowsRead: number;
+  avgRowsReturned: number;
+  rowsWritten: number;
+}
+
+export interface IndexAdviceInput {
+  schema: DataSchema;
+  // Sums over the last 7 days, and over the last 30.
+  week: IndexAdviceStat[];
+  month: IndexAdviceStat[];
+  // Whether stats go back the whole 30 days, so "no query used it" means it.
+  monthCovered: boolean;
+  // "table.column" made filterable, or "table" created, in the last 30 days.
+  recentlyIndexed: Set<string>;
+}
+
+// Indexes the owner can add or remove: a column marked filterable. Unique and
+// relation columns are indexed for what they are, and stay.
+const canIndex = (type: string) => type !== 'json' && type !== 'long_text';
+
+export const adviseIndexes = (
+  input: IndexAdviceInput
+): IndexAdviceCandidate[] => {
+  const out: IndexAdviceCandidate[] = [];
+  const tables = new Map(input.schema.tables.map(t => [t.name, t]));
+
+  for (const stat of input.week) {
+    if (!stat.columnName) continue;
+    const table = tables.get(stat.tableName);
+    const column = table?.columns.find(c => c.name === stat.columnName);
+    if (!column || isIndexedColumn(column) || !canIndex(column.type)) continue;
+    if (stat.queries < constants.DATA_INDEX_ADVICE_MIN_QUERIES) continue;
+    const avgRead = stat.rowsRead / stat.queries;
+    const avgReturned = stat.rowsReturned / stat.queries;
+    if (avgRead < constants.DATA_INDEX_ADVICE_MIN_ROWS_READ) continue;
+    // Scanning, not just a small table: many rows read for each one returned.
+    if (
+      stat.rowsRead <
+      constants.DATA_INDEX_ADVICE_MIN_READ_RATIO * stat.rowsReturned
+    ) {
+      continue;
+    }
+    out.push({
+      tableName: stat.tableName,
+      columnName: stat.columnName,
+      kind: constants.DATA_INDEX_ADVICE_ADD,
+      queries: stat.queries,
+      avgRowsRead: Math.round(avgRead),
+      avgRowsReturned: Math.round(avgReturned),
+      rowsWritten: 0
+    });
+  }
+
+  if (input.monthCovered) {
+    const used = new Set(
+      input.month
+        .filter(s => s.columnName && s.queries > 0)
+        .map(s => `${s.tableName}.${s.columnName}`)
+    );
+    for (const table of input.schema.tables) {
+      const totals = input.month.find(
+        s => s.tableName === table.name && s.columnName === ''
+      );
+      const written = totals?.rowsWritten ?? 0;
+      if (written < constants.DATA_INDEX_UNUSED_MIN_ROWS_WRITTEN) continue;
+      if (written <= (totals?.queries ?? 0)) continue;
+      if (input.recentlyIndexed.has(table.name)) continue;
+      for (const column of table.columns) {
+        if (!column.filterable || column.unique) continue;
+        if (column.type === 'relation' || column.type === 'file') continue;
+        const key = `${table.name}.${column.name}`;
+        if (used.has(key) || input.recentlyIndexed.has(key)) continue;
+        out.push({
+          tableName: table.name,
+          columnName: column.name,
+          kind: constants.DATA_INDEX_ADVICE_UNUSED,
+          queries: 0,
+          avgRowsRead: 0,
+          avgRowsReturned: 0,
+          rowsWritten: written
+        });
+      }
+    }
+  }
+  return out;
+};
+
+// A dismissed suggestion comes back once its numbers have doubled.
+export const adviceDoubled = (
+  current: { queries: number; avgRowsRead: number; rowsWritten: number },
+  dismissed: {
+    queries: number | null;
+    avgRowsRead: number | null;
+    rowsWritten: number | null;
+  }
+): boolean =>
+  (dismissed.queries !== null &&
+    dismissed.queries > 0 &&
+    current.queries >= 2 * dismissed.queries) ||
+  (dismissed.avgRowsRead !== null &&
+    dismissed.avgRowsRead > 0 &&
+    current.avgRowsRead >= 2 * dismissed.avgRowsRead) ||
+  (dismissed.rowsWritten !== null &&
+    dismissed.rowsWritten > 0 &&
+    current.rowsWritten >= 2 * dismissed.rowsWritten);