@ganju/utils 0.0.20 → 0.0.22

package/src/dataImport.ts

@@ -482,6 +482,9 @@ export interface ImportedRow {
   errors: string[];
   // Values read with a guess; the row is still imported.
   warnings: NumberGuess[];
+  // The unique columns whose value an earlier line already has (each also
+  // one of `errors`): a sheet sync lets the database say so instead.
+  repeated: string[];
 }
 
 // Every row of the file as the values it will insert, with each problem
@@ -512,6 +515,7 @@ export const importRows = (
     const values: Record<string, unknown> = {};
     const errors = missing.map(c => `${c.name} is required.`);
     const warnings: NumberGuess[] = [];
+    const repeated: string[] = [];
     mapping.forEach((name, i) => {
       if (name === null) return;
       const column = columns.get(name);
@@ -534,13 +538,14 @@ export const importRows = (
         const first = lines.get(key);
         if (first === undefined) lines.set(key, line);
         else {
+          repeated.push(column.name);
           errors.push(
             `${column.name}: "${text(row[i])}" is already on line ${first}, and this column takes each value once.`
           );
         }
       }
     });
-    return { index, line, values, errors, warnings };
+    return { index, line, values, errors, warnings, repeated };
   });
 };
 
@@ -555,3 +560,13 @@ export const csvLine = (cells: ImportCell[]) =>
 
 export { detectSeparator, parseCsv } from './dataCsv';
 export type { ParsedCsv } from './dataCsv';
+export {
+  DATA_CSV_MIME,
+  DATA_SPREADSHEET_MIME,
+  cellFromSheet,
+  isTabularMimeType,
+  readCsvText,
+  readSheet,
+  readTabularFile
+} from './dataSheet';
+export type { SheetJs, TabularFile } from './dataSheet';

package/src/dataIndexAdvice.ts

@@ -0,0 +1,127 @@
+// Which columns of a database are worth an index, and which indexes nobody
+// uses, from the API's daily query stats and the database's schema. Pure, so
+// the daily job and the tests apply the same rule.
+import { constants } from './constants';
+import { isIndexedColumn } from './dataDdl';
+import type { DataSchema } from './dataTypes';
+
+// A column's (or, with column '', a table's) sums over a window.
+export interface IndexAdviceStat {
+  tableName: string;
+  columnName: string;
+  queries: number;
+  rowsRead: number;
+  rowsReturned: number;
+  rowsWritten: number;
+}
+
+export interface IndexAdviceCandidate {
+  tableName: string;
+  columnName: string;
+  kind: 'add' | 'unused';
+  queries: number;
+  avgRowsRead: number;
+  avgRowsReturned: number;
+  rowsWritten: number;
+}
+
+export interface IndexAdviceInput {
+  schema: DataSchema;
+  // Sums over the last 7 days, and over the last 30.
+  week: IndexAdviceStat[];
+  month: IndexAdviceStat[];
+  // Whether stats go back the whole 30 days, so "no query used it" means it.
+  monthCovered: boolean;
+  // "table.column" made filterable, or "table" created, in the last 30 days.
+  recentlyIndexed: Set<string>;
+}
+
+// Indexes the owner can add or remove: a column marked filterable. Unique and
+// relation columns are indexed for what they are, and stay.
+const canIndex = (type: string) => type !== 'json' && type !== 'long_text';
+
+export const adviseIndexes = (
+  input: IndexAdviceInput
+): IndexAdviceCandidate[] => {
+  const out: IndexAdviceCandidate[] = [];
+  const tables = new Map(input.schema.tables.map(t => [t.name, t]));
+
+  for (const stat of input.week) {
+    if (!stat.columnName) continue;
+    const table = tables.get(stat.tableName);
+    const column = table?.columns.find(c => c.name === stat.columnName);
+    if (!column || isIndexedColumn(column) || !canIndex(column.type)) continue;
+    if (stat.queries < constants.DATA_INDEX_ADVICE_MIN_QUERIES) continue;
+    const avgRead = stat.rowsRead / stat.queries;
+    const avgReturned = stat.rowsReturned / stat.queries;
+    if (avgRead < constants.DATA_INDEX_ADVICE_MIN_ROWS_READ) continue;
+    // Scanning, not just a small table: many rows read for each one returned.
+    if (
+      stat.rowsRead <
+      constants.DATA_INDEX_ADVICE_MIN_READ_RATIO * stat.rowsReturned
+    ) {
+      continue;
+    }
+    out.push({
+      tableName: stat.tableName,
+      columnName: stat.columnName,
+      kind: constants.DATA_INDEX_ADVICE_ADD,
+      queries: stat.queries,
+      avgRowsRead: Math.round(avgRead),
+      avgRowsReturned: Math.round(avgReturned),
+      rowsWritten: 0
+    });
+  }
+
+  if (input.monthCovered) {
+    const used = new Set(
+      input.month
+        .filter(s => s.columnName && s.queries > 0)
+        .map(s => `${s.tableName}.${s.columnName}`)
+    );
+    for (const table of input.schema.tables) {
+      const totals = input.month.find(
+        s => s.tableName === table.name && s.columnName === ''
+      );
+      const written = totals?.rowsWritten ?? 0;
+      if (written < constants.DATA_INDEX_UNUSED_MIN_ROWS_WRITTEN) continue;
+      if (written <= (totals?.queries ?? 0)) continue;
+      if (input.recentlyIndexed.has(table.name)) continue;
+      for (const column of table.columns) {
+        if (!column.filterable || column.unique) continue;
+        if (column.type === 'relation' || column.type === 'file') continue;
+        const key = `${table.name}.${column.name}`;
+        if (used.has(key) || input.recentlyIndexed.has(key)) continue;
+        out.push({
+          tableName: table.name,
+          columnName: column.name,
+          kind: constants.DATA_INDEX_ADVICE_UNUSED,
+          queries: 0,
+          avgRowsRead: 0,
+          avgRowsReturned: 0,
+          rowsWritten: written
+        });
+      }
+    }
+  }
+  return out;
+};
+
+// A dismissed suggestion comes back once its numbers have doubled.
+export const adviceDoubled = (
+  current: { queries: number; avgRowsRead: number; rowsWritten: number },
+  dismissed: {
+    queries: number | null;
+    avgRowsRead: number | null;
+    rowsWritten: number | null;
+  }
+): boolean =>
+  (dismissed.queries !== null &&
+    dismissed.queries > 0 &&
+    current.queries >= 2 * dismissed.queries) ||
+  (dismissed.avgRowsRead !== null &&
+    dismissed.avgRowsRead > 0 &&
+    current.avgRowsRead >= 2 * dismissed.avgRowsRead) ||
+  (dismissed.rowsWritten !== null &&
+    dismissed.rowsWritten > 0 &&
+    current.rowsWritten >= 2 * dismissed.rowsWritten);

package/src/dataRpc.ts

@@ -4,11 +4,16 @@
 import type {
   DataCaller,
   DataDeleteImpact,
+  DataFileLinkChange,
+  DataFileLinks,
+  DataFileLinksReplaced,
   DataResult,
   DataSchema,
   DataTable,
-  DataTableCopy
+  DataTableCopy,
+  DataTableSync
 } from './dataTypes';
+import type { SheetSyncArgs } from './dataEngine';
 import type { DataQuery } from './dataQueries';
 import type { DataTrigger, DataTriggerWithSecret, SavedTrigger } from './dataTriggers';
 
@@ -56,7 +61,8 @@ export interface DatabaseInfo {
   timeZone: string | null;
   schemaVersion: number;
   size: number;
-  tables: { name: string; label: string | null; columns: number }[];
+  // `rows` is about how many: the highest id, read from the index.
+  tables: { name: string; label: string | null; columns: number; rows: number }[];
   // A column change copying a large table in the background: table changes,
   // and writes that would change its rows, wait while it runs.
   copy: DataTableCopy | null;
@@ -106,6 +112,12 @@ export interface ProjectDatabaseRpc {
   dismissCopy(caller: DataCaller): R<{ copy: null }>;
   describe(caller: DataCaller, args?: unknown): R<unknown>;
   query(caller: DataCaller, args: unknown): R<unknown>;
+  // A row as a trigger event carries it (`id`, or the newest): for a test
+  // send and for a run whose event was too large to carry its row.
+  eventRow(
+    caller: DataCaller,
+    args: { table: string; id?: number }
+  ): R<Record<string, unknown> | null>;
   write(
     caller: DataCaller,
     args: unknown
@@ -114,6 +126,19 @@ export interface ProjectDatabaseRpc {
     caller: DataCaller,
     args: { table: string; ids?: number[] }
   ): R<DataDeleteImpact>;
+  // The rows that link knowledge items, and moving or clearing those links.
+  fileLinks(caller: DataCaller, args: { uris: string[] }): R<DataFileLinks>;
+  replaceFileLinks(
+    caller: DataCaller,
+    args: { changes: DataFileLinkChange[] }
+  ): R<DataFileLinksReplaced>;
+  // A synced table's rows made what its sheet says, all or nothing; and why
+  // a sync that never reached the rows was refused.
+  syncRows(caller: DataCaller, args: SheetSyncArgs): R<DataTableSync>;
+  recordSyncFailure(
+    caller: DataCaller,
+    args: { table: string; error: string; title?: string | null; version?: string | null }
+  ): R<DataTableSync | null>;
   history(
     caller: DataCaller,
     args: { table: string; id: number; limit?: number; cursor?: number }

package/src/dataSheet.ts

@@ -0,0 +1,147 @@
+// Reading a spreadsheet or CSV from the project's knowledge into header and
+// rows, for data-admin's import and a table's sheet sync alike. SheetJS is
+// passed in, so each worker loads it only when a file needs it.
+import { parseCsv } from './dataCsv';
+import { DATA_SHEET_MAX_BYTES, DataError } from './dataTypes';
+import type { ImportCell } from './dataImport';
+
+export const DATA_SPREADSHEET_MIME =
+  /spreadsheetml|ms-excel|opendocument\.spreadsheet/i;
+export const DATA_CSV_MIME = /csv|text\/plain|tab-separated/i;
+
+// Whether a knowledge item can be a table's source.
+export const isTabularMimeType = (mimeType: string | null | undefined) =>
+  !!mimeType &&
+  (DATA_SPREADSHEET_MIME.test(mimeType) || DATA_CSV_MIME.test(mimeType));
+
+// The parts of SheetJS used here, so this package doesn't depend on it.
+export interface SheetJs {
+  read(
+    data: ArrayBuffer,
+    options: Record<string, unknown>
+  ): {
+    SheetNames: string[];
+    Sheets: Record<string, unknown>;
+  };
+  utils: {
+    sheet_to_json<T>(sheet: unknown, options: Record<string, unknown>): T[];
+    decode_range(ref: string): { s: { r: number } };
+  };
+}
+
+export interface TabularFile {
+  headers: string[];
+  rows: ImportCell[][];
+  // Each row's line in the file or sheet, counting the header.
+  lines: number[];
+}
+
+const pad = (n: number) => String(n).padStart(2, '0');
+
+// A spreadsheet's own date as the text the database reads, as the
+// dashboard's import does: a date alone when it has no time, else a
+// wall-clock time.
+export const cellFromSheet = (cell: unknown): ImportCell => {
+  if (cell instanceof Date) {
+    const date = `${cell.getFullYear()}-${pad(cell.getMonth() + 1)}-${pad(cell.getDate())}`;
+    if (!cell.getHours() && !cell.getMinutes() && !cell.getSeconds()) {
+      return date;
+    }
+    return `${date}T${pad(cell.getHours())}:${pad(cell.getMinutes())}:${pad(cell.getSeconds())}`;
+  }
+  return typeof cell === 'string' ||
+    typeof cell === 'number' ||
+    typeof cell === 'boolean'
+    ? cell
+    : null;
+};
+
+// One sheet of an Excel or OpenDocument file: the first unless named.
+// Blank lines are left out.
+export const readSheet = (
+  XLSX: SheetJs,
+  bytes: ArrayBuffer,
+  uri: string,
+  sheetName?: string | null
+): TabularFile => {
+  // Only the sheet being read is parsed: a workbook unpacks to many times
+  // its size, and a worker has 128 MiB.
+  const book = XLSX.read(bytes, {
+    cellDates: true,
+    dense: true,
+    sheets: sheetName ?? 0
+  });
+  const name = sheetName ?? book.SheetNames[0];
+  const sheet = name ? book.Sheets[name] : undefined;
+  if (!sheet) {
+    throw new DataError(
+      `${uri} has no sheet "${sheetName}". Sheets: ${book.SheetNames.join(', ')}.`,
+      'not_found'
+    );
+  }
+  const all = XLSX.utils.sheet_to_json<unknown[]>(sheet, {
+    header: 1,
+    raw: true,
+    defval: null,
+    blankrows: true
+  });
+  // The sheet's range may start below row 1.
+  const ref = (sheet as { '!ref'?: string })['!ref'];
+  const firstLine = ref ? XLSX.utils.decode_range(ref).s.r + 1 : 1;
+  const [header = [], ...body] = all;
+  const rows: ImportCell[][] = [];
+  const lines: number[] = [];
+  body.forEach((row, i) => {
+    const cells = row.map(cellFromSheet);
+    if (cells.every(c => c === null || c === '')) return;
+    rows.push(cells);
+    lines.push(firstLine + 1 + i);
+  });
+  return {
+    headers: header.map(h => (h === null ? '' : String(h))),
+    rows,
+    lines
+  };
+};
+
+// A CSV's text, with each row's line as a sheet's would be.
+export const readCsvText = (text: string): TabularFile => {
+  const parsed = parseCsv(text);
+  return {
+    headers: parsed.headers,
+    rows: parsed.rows,
+    lines: parsed.lines
+  };
+};
+
+// A stored file, read by its type: a spreadsheet through SheetJS (loaded
+// only then), anything else as CSV text. Over DATA_SHEET_MAX_BYTES it isn't
+// read at all.
+export const readTabularFile = async (file: {
+  bytes: ArrayBuffer;
+  mimeType: string | null;
+  uri: string;
+  sheet?: string | null;
+  loadSheetJs: () => Promise<SheetJs>;
+}): Promise<TabularFile> => {
+  if (file.bytes.byteLength > DATA_SHEET_MAX_BYTES) {
+    throw new DataError(
+      `${file.uri} is over ${DATA_SHEET_MAX_BYTES / (1024 * 1024)} MB.`,
+      'too_large'
+    );
+  }
+  if (file.mimeType && DATA_SPREADSHEET_MIME.test(file.mimeType)) {
+    return readSheet(
+      await file.loadSheetJs(),
+      file.bytes,
+      file.uri,
+      file.sheet
+    );
+  }
+  if (!file.mimeType || DATA_CSV_MIME.test(file.mimeType)) {
+    return readCsvText(new TextDecoder().decode(file.bytes));
+  }
+  throw new DataError(
+    `${file.uri} is ${file.mimeType}; a table reads CSV and Excel files.`
+  );
+};

package/src/dataTools.ts

@@ -61,7 +61,7 @@ const CHANGES = {
 };
 
 const WHERE_HELP =
-  'Filter object. { "column": value } matches exactly; { "column": { "op": value } } with op one of eq, ne, gt, gte, lt, lte, in, notIn (lists), contains, startsWith (text, case-insensitive), isNull (true/false). Several keys must all match; { "or": [ {...}, {...} ] } matches any. A relation column takes the related row\'s id, or filter on a column of the related table with "relation.column", e.g. { "product.name": "Margherita" }. Datetimes are ISO 8601; include the offset for local times, e.g. "2026-10-19T00:00:00-05:00".';
+  'Filter object. { "column": value } matches exactly; { "column": { "op": value } } with op one of eq, ne, gt, gte, lt, lte, in, notIn (lists), contains, startsWith (text, case-insensitive), isNull (true/false). Several keys must all match; { "or": [ {...}, {...} ] } matches any. A relation column takes the related row\'s id, or filter on a column of the related table with "relation.column", e.g. { "product.name": "Margherita" }. A file column matches the item\'s uri. Datetimes are ISO 8601; include the offset for local times, e.g. "2026-10-19T00:00:00-05:00".';
 
 const accessOf = (caller: DataCaller, table: DataTable) =>
   caller.restricted ? table.access : 'full';
@@ -145,7 +145,7 @@ export const dataToolDefinitions = (
       title: DATA_TOOL_TITLES[DATA_TOOL_QUERY],
       description: [
         "Read rows from the project's database: structured records such as products, prices, stock, orders, bookings or customers. Use it for anything with fields and numbers; use the knowledge/resource tools for documents and pages.",
-        'Returns matching rows (relation columns come back as { id, label }), the total that match, and nextCursor when there are more. With "aggregate" it returns counts, sums, averages, minimums or maximums instead of rows, optionally per group: use that for "how many" and "how much" rather than adding rows up yourself.',
+        'Returns matching rows (relation columns come back as { id, label }; file columns as { uri, title, mimeType }, or { uri, missing: true } when the item is gone: deliver one with send-resource and its uri, or read a document with read-resource), the total that match, and nextCursor when there are more. With "aggregate" it returns counts, sums, averages, minimums or maximums instead of rows, optionally per group: use that for "how many" and "how much" rather than adding rows up yourself.',
         "The database holds only what these tables and columns hold. When a question needs something they don't have (opening hours, parking, calories, reviews, forecasts), look in the knowledge tools if you have them; if nothing has it, say you don't have that information. Never guess it.",
         `Every table also has id, created_at and updated_at. Datetimes come back in the project's time zone (${timeZone}) with their offset; read them as local times.`,
         'Tables:',
@@ -242,7 +242,7 @@ export const dataToolDefinitions = (
       title: DATA_TOOL_TITLES[DATA_TOOL_WRITE],
       description: [
         "Add, change or delete rows in the project's database. All operations in one call succeed together or none is applied, so put everything that belongs together (an order, its lines and the stock they use) in one call.",
-        '- insert: { "op": "insert", "table": "...", "values": { column: value } }. A relation column takes the related row\'s id; to point at a row inserted earlier in the same call use { "$ref": n }, n being that operation\'s position (0-based).',
+        '- insert: { "op": "insert", "table": "...", "values": { column: value } }. A relation column takes the related row\'s id; to point at a row inserted earlier in the same call use { "$ref": n }, n being that operation\'s position (0-based). A file column takes the uri of an item in the project\'s knowledge (from list-resources or search-resources), or null.',
         '- update: { "op": "update", "table": "...", "id": 12, "set": { column: value } }. For numbers, { "decrement": 2 } or { "increment": 2 } changes the value relative to what is stored. Instead of id you can give "where" (same format as data-query).',
         '- delete: { "op": "delete", "table": "...", "id": 12 } (or "where").',
         'Guards: an update or delete also takes "where" conditions that must hold, e.g. "where": { "stock": { "gte": 2 } }, and "expect", how many rows must change (1 by default when id is given). If a guard fails, nothing in the call is written and the error says why (e.g. how much stock is left).',