@wirecat/cli-core 0.451.0 → 0.453.0

package/dist/store/sqlite/event-search.js

@@ -0,0 +1,163 @@
+import { QUERY_LIMITS } from "../../search/lucene/types.js";
+import { eventOf } from "./events.js";
+import { CORPORA, corpusIndexState, drainCorpus, EVENT_LIVE, EVENT_TEXT, noteIndexText, searchableCopy, } from "./note-index.js";
+import { searchCorpus } from "./note-search.js";
+import { stemmerCache } from "./stems.js";
+import { read } from "./transaction.js";
+import { dot } from "./vectors.js";
+/** The fields an event has; a message or note field is refused rather than matching nothing. */
+export const EVENT_FIELDS = ["text", "exact", "body", "title", "date", "in"];
+/**
+ * Whether the event `event` (an SQL expression of its id) has a searchable copy in the accounts of the one parameter,
+ * a JSON list of stored account ids. Words and meaning both filter by it, so an event held in several accounts is
+ * found through each of them.
+ */
+const inEventAccounts = (event) => searchableCopy(event, "c.account_id IN (SELECT CAST(value AS INTEGER) FROM json_each(?))");
+const EVENT_CORPUS = {
+    type: "event",
+    noun: "event",
+    fields: EVENT_FIELDS,
+    view: `(SELECT e.id, NULL AS folder_id, NULL AS path, e.title, ${EVENT_TEXT} AS text, NULL AS metadata, ` +
+        `e.starts_at AS at, e.deleted_at, ${EVENT_LIVE} AS live FROM events e)`,
+    words: "event_words",
+    stems: "event_stems",
+    live: "n.live = 1",
+    inAccounts: (ids) => ({
+        sql: inEventAccounts("n.id"),
+        params: [ids],
+        exact: true,
+    }),
+};
+const SCAN_PAGE = 5_000;
+export const eventSearchOver = (context) => {
+    const { database } = context;
+    const stemmerFor = stemmerCache();
+    const drain = () => drainCorpus(database, CORPORA.event, stemmerFor);
+    /** The live event pieces with a searchable copy in these accounts, as SQL over chunks `k`. */
+    const live = (accountIds) => {
+        const sql = ` AND k.chunkable_type = 'event' AND EXISTS (SELECT 1 FROM events e WHERE e.id = k.chunkable_id AND ${EVENT_LIVE})`;
+        return accountIds?.length
+            ? { sql: `${sql} AND ${inEventAccounts("k.chunkable_id")}`, params: [JSON.stringify(accountIds)] }
+            : { sql, params: [] };
+    };
+    /** Events in rank order, as hits: one per series, at its occurrence nearest now that is not cancelled. */
+    const grouped = (rows, accountIds) => {
+        const facts = new Map(database
+            .prepare("SELECT id, event_series_id AS series, starts_at, status FROM events WHERE id IN (SELECT value FROM json_each(?))")
+            .all(JSON.stringify(rows.map(({ id }) => id)))
+            .map((row) => [Number(row.id), row]));
+        const groups = new Map();
+        for (const row of rows) {
+            const fact = facts.get(row.id);
+            if (!fact)
+                continue;
+            const key = fact.series == null ? `e${row.id}` : `s${fact.series}`;
+            const group = groups.get(key) ?? { first: row, members: [] };
+            group.members.push(fact);
+            groups.set(key, group);
+        }
+        const now = context.now();
+        const nearest = (members) => members.toSorted((a, b) => Number(a.status === "cancelled") - Number(b.status === "cancelled") ||
+            Math.abs(Number(a.starts_at ?? 0) - now) - Math.abs(Number(b.starts_at ?? 0) - now) ||
+            Number(a.id) - Number(b.id))[0];
+        const event = database.prepare(`SELECT e.*, ${EVENT_TEXT} AS text FROM events e WHERE e.id = ?`);
+        const place = database.prepare("SELECT a.provider, a.external_id AS account, ce.url FROM calendar_entries ce " +
+            "JOIN calendars c ON c.id = ce.calendar_id JOIN accounts a ON a.id = c.account_id " +
+            "WHERE ce.event_id = ? " +
+            "ORDER BY c.account_id NOT IN (SELECT CAST(value AS INTEGER) FROM json_each(?)), ce.deleted_at IS NOT NULL, ce.id LIMIT 1");
+        const chosen = JSON.stringify(accountIds ?? []);
+        return [...groups.values()].map(({ first, members }) => {
+            const id = Number(nearest(members).id);
+            const row = event.get(id);
+            const at = place.get(id, chosen);
+            return {
+                first,
+                hit: {
+                    ref: `event:${id}`,
+                    event: eventOf(row),
+                    text: String(row.text ?? ""),
+                    occurrences: members.length,
+                    account: at ? { provider: String(at.provider), account: String(at.account) } : null,
+                    url: at?.url == null ? null : String(at.url),
+                },
+            };
+        });
+    };
+    return {
+        search: async (query) => {
+            drain();
+            const offset = query.offset ?? 0;
+            const candidates = query.candidates ?? QUERY_LIMITS.candidates;
+            const matched = searchCorpus(context, { ...query, limit: candidates, offset: 0 }, EVENT_CORPUS);
+            const truncated = matched.length > candidates;
+            const rows = truncated ? matched.slice(0, candidates) : matched;
+            return read(database, () => {
+                const groups = grouped(rows, query.accountIds);
+                return {
+                    items: groups.slice(offset, offset + query.limit).map(({ first, hit }) => ({
+                        ...hit,
+                        relevance: first.relevance,
+                        ...(first.exact === undefined ? {} : { exact: first.exact }),
+                    })),
+                    hasMore: groups.length > offset + query.limit,
+                    truncated,
+                };
+            });
+        },
+        eventIndexState: async () => corpusIndexState(database, CORPORA.event),
+        chunksToEmbed: async (model, { after, limit }) => {
+            drain();
+            const scope = live(undefined);
+            const text = database.prepare(`SELECT e.title, ${EVENT_TEXT} AS body FROM events e WHERE e.id = ?`);
+            return database
+                .prepare(`SELECT k.content_hash AS hash, min(k.chunkable_id) AS owner, k.start_offset AS start, k.end_offset AS end
+             FROM chunks k
+            WHERE k.content_hash > ?${scope.sql}
+              AND NOT EXISTS (SELECT 1 FROM embeddings v WHERE v.model = ? AND v.content_hash = k.content_hash)
+            GROUP BY k.content_hash ORDER BY k.content_hash LIMIT ?`)
+                .all(after ?? "", ...scope.params, model, limit)
+                .flatMap((row) => {
+                const event = text.get(Number(row.owner));
+                if (!event)
+                    return [];
+                const whole = noteIndexText(event.title == null ? null : String(event.title), String(event.body));
+                return [{ hash: String(row.hash), text: whole.slice(Number(row.start), Number(row.end)) }];
+            });
+        },
+        embedded: async (model, { accountIds } = {}) => {
+            drain();
+            const scope = live(accountIds);
+            return (database
+                .prepare(`SELECT 1 FROM chunks k JOIN embeddings v ON v.model = ? AND v.content_hash = k.content_hash
+              WHERE 1 = 1${scope.sql} LIMIT 1`)
+                .get(model, ...scope.params) !== undefined);
+        },
+        nearest: async (model, query, { limit, accountIds }) => {
+            drain();
+            const scope = live(accountIds);
+            const page = database.prepare(`SELECT k.id, k.chunkable_id AS owner, v.vector
+           FROM chunks k JOIN embeddings v ON v.model = ? AND v.content_hash = k.content_hash
+          WHERE k.id > ?${scope.sql}
+          ORDER BY k.id LIMIT ?`);
+            const best = new Map();
+            let after = 0;
+            for (;;) {
+                const rows = page.all(model, after, ...scope.params, SCAN_PAGE);
+                for (const row of rows) {
+                    const score = dot(query, row.vector);
+                    const owner = Number(row.owner);
+                    if (score > (best.get(owner) ?? Number.NEGATIVE_INFINITY))
+                        best.set(owner, score);
+                }
+                const last = rows.at(-1);
+                if (!last || rows.length < SCAN_PAGE)
+                    break;
+                after = Number(last.id);
+            }
+            const ranked = [...best.entries()].map(([id, score]) => ({ id, score })).sort((a, b) => b.score - a.score);
+            return read(database, () => grouped(ranked, accountIds)
+                .slice(0, limit)
+                .map(({ first, hit }) => ({ ...hit, score: first.score })));
+        },
+    };
+};

package/dist/store/sqlite/files.d.ts

@@ -15,6 +15,10 @@ interface Placed {
 export declare const logFileLink: (database: CacheDatabase, now: number, fileId: number, from: unknown, to: number | null, method?: string, author?: string) => {
     changes: number;
 };
+/** How an `attached` link names a Drive file it waits for: by its Drive id, since no row may hold it yet. */
+export declare const driveFileTarget: (fileId: string) => string;
+/** The document of the Drive file with this id; `null` when no Drive account holds it, or two do. */
+export declare const driveDocumentOf: (database: CacheDatabase, fileId: string) => number | null;
 /**
  * Writes the `files` row of one document from the document as it is now: the document's source record. A Drive
  * document (catalog metadata or Drive front matter) is keyed by its Drive file id and placed under its first parent;

package/dist/store/sqlite/files.js

@@ -48,6 +48,30 @@ export const logFileLink = (database, now, fileId, from, to, method = "import",
     .prepare("INSERT INTO entity_link_events (linkable_type, linkable_id, entity_type, from_entity_id, to_entity_id, method, " +
     "confidence, author, created_at) VALUES ('file', ?, 'document', ?, ?, ?, 1, ?, ?)")
     .run(fileId, from == null ? null : Number(from), to, method, author, now);
+/** How an `attached` link names a Drive file it waits for: by its Drive id, since no row may hold it yet. */
+export const driveFileTarget = (fileId) => `google-drive:${fileId}`;
+/** The document of the Drive file with this id; `null` when no Drive account holds it, or two do. */
+export const driveDocumentOf = (database, fileId) => {
+    const rows = database
+        .prepare("SELECT DISTINCT f.document_id AS id FROM files f JOIN accounts a ON a.id = f.account_id WHERE a.provider = 'google-drive' " +
+        "AND f.external_id = ? AND f.deleted_at IS NULL AND f.document_id IS NOT NULL LIMIT 2")
+        .all(fileId);
+    return rows.length === 1 ? Number(rows[0]?.id) : null;
+};
+/** Calendar entries that named this Drive file before Drive stored it now link its document. */
+const linkAttached = (database, now, fileId) => {
+    const target = driveFileTarget(fileId);
+    if (!database
+        .prepare("SELECT 1 FROM links WHERE to_type = 'document' AND to_id IS NULL AND target_text = ? AND kind = 'attached'")
+        .get(target))
+        return;
+    const document = driveDocumentOf(database, fileId);
+    if (document !== null)
+        database
+            .prepare("UPDATE links SET to_id = ?, updated_at = ? WHERE to_type = 'document' AND to_id IS NULL AND target_text = ? " +
+            "AND kind = 'attached'")
+            .run(document, now, target);
+};
 /** Moves the rows under a folder whose path changed, in one statement over the old prefix. */
 const movePaths = (database, accountId, from, to) => {
     database
@@ -136,6 +160,8 @@ export const syncFile = (database, now, documentId, { seen = true, depth = 0, pl
             .get(...Object.values(values), Number(doc.created_at), now);
         const placed = { id: Number(row?.id), path };
         logFileLink(database, now, placed.id, null, documentId, method);
+        if (remote)
+            linkAttached(database, now, externalId);
         if (remote && kind === "folder")
             placeChildren(database, now, accountId, String(doc.external_id), depth);
         return placed;
@@ -152,6 +178,8 @@ export const syncFile = (database, now, documentId, { seen = true, depth = 0, pl
             .run(...Object.values(values), changed ? now : Number(existing.updated_at), id);
         if (kind === "folder" && existing.path !== path)
             movePaths(database, accountId, String(existing.path), path);
+        if (remote && changed)
+            linkAttached(database, now, externalId);
     }
     return { id, path };
 };

package/dist/store/sqlite/link-kinds.d.ts

@@ -2,8 +2,12 @@
  * Every kind a `links` row may have. The schema doc's description of `links` names exactly these
  * (`schema.test.ts` checks it); a writer that needs a new kind adds it here first.
  */
-export declare const LINK_KINDS: readonly ["links-to", "blocked-by", "follow-up-of", "about", "member-of", "labelled", "answered-by", "evidence", "created-from", "duplicate-of", "related-to", "assigned-to"];
+export declare const LINK_KINDS: readonly ["links-to", "blocked-by", "follow-up-of", "about", "member-of", "labelled", "answered-by", "evidence", "created-from", "duplicate-of", "related-to", "assigned-to", "attached"];
 export type LinkKind = (typeof LINK_KINDS)[number];
+/** Where a caller may say a link came from. */
+export declare const LINK_ORIGINS: readonly ["file", "owner", "suggested", "message"];
+/** Where a stored link came from: an origin a caller gives, or `calendar`, which only the store writes. */
+export declare const LINK_SOURCES: readonly ["file", "owner", "suggested", "message", "calendar"];
 /** The kinds `knowledge.relate` makes between people, organizations and projects. */
 export declare const RELATION_KINDS: readonly ["member-of", "related-to"];
 /** A kind as an SQL literal, so a kind written into a query is checked against the list. */

package/dist/store/sqlite/link-kinds.js

@@ -15,7 +15,12 @@ export const LINK_KINDS = [
     "duplicate-of",
     "related-to",
     "assigned-to",
+    "attached",
 ];
+/** Where a caller may say a link came from. */
+export const LINK_ORIGINS = ["file", "owner", "suggested", "message"];
+/** Where a stored link came from: an origin a caller gives, or `calendar`, which only the store writes. */
+export const LINK_SOURCES = [...LINK_ORIGINS, "calendar"];
 /** The kinds `knowledge.relate` makes between people, organizations and projects. */
 export const RELATION_KINDS = ["member-of", "related-to"];
 /** A kind as an SQL literal, so a kind written into a query is checked against the list. */

package/dist/store/sqlite/note-index.d.ts

@@ -31,6 +31,23 @@ export interface Corpus {
         stems: string;
     };
 }
+/** What an event `e` is searched by besides its title: the cleaned description, the place and the attendees' names. */
+export declare const EVENT_TEXT: string;
+/** Calendar entry types that are listed, never searched (requirements 3.4). */
+export declare const UNSEARCHED_EVENT_TYPES: readonly ["workingLocation", "focusTime", "outOfOffice", "birthday"];
+/** Visibilities whose entry, when it carries no text, is a busy-only slot: stored, never searched (requirements 3.4). */
+export declare const BUSY_ONLY_VISIBILITIES: readonly ["private", "confidential"];
+/**
+ * Whether the event `event` (an SQL expression of its id) has a searchable calendar copy, and with `account` one in
+ * those accounts (an SQL condition over `c.account_id`). Only live copies count, unless none is: an event its
+ * calendars deleted keeps its past copies as history.
+ */
+export declare const searchableCopy: (event: string, account?: string) => string;
+/**
+ * Whether an event `e` is searched at all: by words, by meaning and in `search all`. The one place that names what
+ * is left out. An event no calendar holds (a Zoom meeting's own) is searched; a cancelled one is not.
+ */
+export declare const EVENT_LIVE: string;
 export declare const CORPORA: {
     readonly document: {
         readonly type: "document";
@@ -81,7 +98,6 @@ export declare const CORPORA: {
         readonly index: "event_index";
         readonly table: "events";
         readonly read: string;
-        readonly pieces: false;
     };
 };
 /** What a row is indexed as: its title, then its text. Chunk offsets point into this. */

package/dist/store/sqlite/note-index.js

@@ -9,6 +9,34 @@ import { speech } from "./lucene-fields.js";
 import { savedStemmers } from "./stems.js";
 import { write } from "./transaction.js";
 import { dropUnusedVectors } from "./vector-index.js";
+/** What an event `e` is searched by besides its title: the cleaned description, the place and the attendees' names. */
+export const EVENT_TEXT = "trim(coalesce(e.description, '') || ' ' || coalesce(e.location, '') || ' ' || coalesce((" +
+    "SELECT group_concat(p.name, ' ') FROM calendar_entries ce JOIN calendar_entry_participants p ON p.entry_id = ce.id " +
+    "WHERE ce.event_id = e.id AND ce.deleted_at IS NULL AND p.resource = 0 AND p.name IS NOT NULL), ''))";
+/** Calendar entry types that are listed, never searched (requirements 3.4). */
+export const UNSEARCHED_EVENT_TYPES = ["workingLocation", "focusTime", "outOfOffice", "birthday"];
+/** Visibilities whose entry, when it carries no text, is a busy-only slot: stored, never searched (requirements 3.4). */
+export const BUSY_ONLY_VISIBILITIES = ["private", "confidential"];
+const sqlList = (values) => values.map((value) => `'${value}'`).join(", ");
+/** A calendar copy `ce` that may be searched: of a searchable type, and not a busy-only slot. */
+const SEARCHABLE_COPY = `(ce.event_type IS NULL OR ce.event_type NOT IN (${sqlList(UNSEARCHED_EVENT_TYPES)})) ` +
+    `AND NOT (coalesce(ce.visibility, '') IN (${sqlList(BUSY_ONLY_VISIBILITIES)}) AND trim(coalesce(ce.title, '')) = '' ` +
+    "AND trim(coalesce(ce.description, '')) = '' AND trim(coalesce(ce.location, '')) = '')";
+/**
+ * Whether the event `event` (an SQL expression of its id) has a searchable calendar copy, and with `account` one in
+ * those accounts (an SQL condition over `c.account_id`). Only live copies count, unless none is: an event its
+ * calendars deleted keeps its past copies as history.
+ */
+export const searchableCopy = (event, account) => `EXISTS (SELECT 1 FROM calendar_entries ce${account ? " JOIN calendars c ON c.id = ce.calendar_id" : ""} ` +
+    `WHERE ce.event_id = ${event} AND ${SEARCHABLE_COPY}${account ? ` AND ${account}` : ""} ` +
+    `AND (ce.deleted_at IS NULL OR NOT EXISTS (SELECT 1 FROM calendar_entries l WHERE l.event_id = ${event} ` +
+    "AND l.deleted_at IS NULL)))";
+/**
+ * Whether an event `e` is searched at all: by words, by meaning and in `search all`. The one place that names what
+ * is left out. An event no calendar holds (a Zoom meeting's own) is searched; a cancelled one is not.
+ */
+export const EVENT_LIVE = "(e.deleted_at IS NULL AND coalesce(e.status, '') <> 'cancelled' AND " +
+    `(NOT EXISTS (SELECT 1 FROM calendar_entries ce WHERE ce.event_id = e.id) OR ${searchableCopy("e.id")}))`;
 export const CORPORA = {
     document: {
         type: "document",
@@ -60,15 +88,11 @@ export const CORPORA = {
         type: "event",
         index: "event_index",
         table: "events",
-        // Working location, focus time, out of office and birthdays are listed, never searched (requirements 3.4).
-        read: "SELECT e.title, trim(coalesce(e.description, '') || ' ' || coalesce(e.location, '') || ' ' || coalesce((" +
-            "SELECT group_concat(p.name, ' ') FROM calendar_entries ce JOIN calendar_entry_participants p ON p.entry_id = ce.id " +
-            "WHERE ce.event_id = e.id AND ce.deleted_at IS NULL AND p.resource = 0 AND p.name IS NOT NULL), '')) AS body, " +
-            "e.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM calendar_entries ce WHERE ce.event_id = e.id " +
-            "AND ce.deleted_at IS NULL AND ce.event_type IN ('workingLocation', 'focusTime', 'outOfOffice', 'birthday')) AS live, " +
+        read: `SELECT e.title, ${EVENT_TEXT} AS body, ${EVENT_LIVE} AS live, ` +
             "coalesce((SELECT a.scope FROM calendar_entries ce JOIN calendars c ON c.id = ce.calendar_id " +
-            "JOIN accounts a ON a.id = c.account_id WHERE ce.event_id = e.id LIMIT 1), 'personal') AS scope FROM events e WHERE e.id = ?",
-        pieces: false,
+            "JOIN accounts a ON a.id = c.account_id WHERE ce.event_id = e.id LIMIT 1), 'personal') AS scope, " +
+            "(SELECT c.account_id FROM calendar_entries ce JOIN calendars c ON c.id = ce.calendar_id " +
+            "WHERE ce.event_id = e.id AND ce.deleted_at IS NULL ORDER BY ce.id LIMIT 1) AS account_id FROM events e WHERE e.id = ?",
     },
 };
 /** What a row is indexed as: its title, then its text. Chunk offsets point into this. */

package/dist/store/sqlite/note-search.d.ts

@@ -1,5 +1,6 @@
 import type { ResolvedNode } from "../../search/lucene/resolved.js";
 import type { Stemmer } from "../../search/stem.js";
+import type { SqlValue } from "../driver.js";
 import { type DocumentPictureStore } from "./document-pictures.js";
 import { type NoteIndexState } from "./note-index.js";
 import { type Note } from "./notes.js";
@@ -28,11 +29,52 @@ export interface NoteHit {
 }
 /** The fields a note has. A message-only field (`from:`, `chat:`, …) is refused rather than matching nothing. */
 export declare const NOTE_FIELDS: readonly ["text", "exact", "body", "tag", "date", "in", "has", "title", "site", "lang", "author", "keyword", "source"];
+interface Fragment {
+    sql: string;
+    params: SqlValue[];
+    /** False when the SQL only narrows the candidates and a JS test decides. */
+    exact: boolean;
+    fts?: string;
+    stems?: string;
+}
+/**
+ * Files in notes folders, the notes the owner wrote and calendar events: one query language, each in its own table
+ * and index.
+ */
+export interface Corpus {
+    type: "document" | "note" | "event";
+    /** What a hit is called in an error and in `in:` (`in:notes`, `in:events`). */
+    noun: "note" | "event";
+    fields: readonly string[];
+    /** The rows as `n`, with the columns the query reads under one set of names; `at` is the time `date:` reads. */
+    view: string;
+    words: string;
+    stems: string;
+    /** The rows a search may return at all, besides `deleted_at`. */
+    live?: string;
+    /** The rows of these accounts, as a JSON list of ids. */
+    inAccounts?: (ids: string) => Fragment;
+    /** The tagged rows; a document is also labelled through its folder or a subfolder of it. */
+    tagged?: (tag: string) => Fragment;
+}
+export interface CorpusRow {
+    type: Corpus["type"];
+    id: number;
+    /** The time `date:` reads: a note's last change, an event's start. */
+    at: number | null;
+    relevance: number | null;
+    exact?: boolean;
+}
+/** What one corpus is searched by; `accountIds` narrows a corpus that has accounts. */
+export type CorpusQuery = Pick<NoteQuery, "root" | "limit" | "offset" | "newest" | "stemmer" | "signal"> & {
+    accountIds?: string[];
+};
 /** Notes matching a parsed query, by the same language as messages: files and written notes, merged. */
 export declare const searchNotes: (context: StoreContext, query: NoteQuery) => {
     items: NoteHit[];
     hasMore: boolean;
 };
+export declare const searchCorpus: (context: StoreContext, query: CorpusQuery, corpus: Corpus) => CorpusRow[];
 export interface NoteChunkToEmbed {
     hash: string;
     text: string;
@@ -75,3 +117,4 @@ export interface NoteSearch extends DocumentPictureStore {
     }): Promise<NearestNote[]>;
 }
 export declare const noteSearchOver: (context: StoreContext) => NoteSearch;
+export {};

package/dist/store/sqlite/note-search.js

@@ -40,9 +40,13 @@ const combine = (parts, operator) => ({
 const CORPORA = [
     {
         type: "document",
-        view: "(SELECT id, account_id AS folder_id, external_id AS path, title, coalesce(body, '') AS text, metadata, updated_at, deleted_at FROM documents)",
+        noun: "note",
+        fields: NOTE_FIELDS,
+        view: "(SELECT id, account_id AS folder_id, external_id AS path, title, coalesce(body, '') AS text, metadata, updated_at AS at, deleted_at FROM documents)",
         words: "document_words",
         stems: "document_stems",
+        live: "coalesce(json_extract(n.metadata, '$.catalog.active'), 1)=1",
+        inAccounts: (ids) => bound("n.folder_id IN (SELECT CAST(value AS INTEGER) FROM json_each(?))", ids),
         tagged: (tag) => bound("(n.id IN (SELECT g.taggable_id FROM taggings g JOIN resolved_tags t ON t.id=g.tag_id WHERE t.name = ? AND g.taggable_type = 'document') " +
             "OR n.folder_id IN (SELECT g.taggable_id FROM taggings g JOIN resolved_tags t ON t.id=g.tag_id WHERE t.name = ? AND g.taggable_type = 'account') " +
             "OR EXISTS (SELECT 1 FROM links l JOIN resolved_tags t ON t.id=l.to_id WHERE l.from_type = 'account' AND l.from_id = n.folder_id " +
@@ -50,7 +54,9 @@ const CORPORA = [
     },
     {
         type: "note",
-        view: "(SELECT id, NULL AS folder_id, NULL AS path, title, body AS text, NULL AS metadata, updated_at, deleted_at FROM notes)",
+        noun: "note",
+        fields: NOTE_FIELDS,
+        view: "(SELECT id, NULL AS folder_id, NULL AS path, title, body AS text, NULL AS metadata, updated_at AS at, deleted_at FROM notes)",
         words: "note_words",
         stems: "note_stems",
         tagged: (tag) => bound("n.id IN (SELECT g.taggable_id FROM taggings g JOIN resolved_tags t ON t.id=g.tag_id WHERE t.name = ? AND g.taggable_type = 'note')", tag),
@@ -61,14 +67,14 @@ export const searchNotes = (context, query) => {
     const offset = query.offset ?? 0;
     const chosen = CORPORA.filter(({ type }) => (query.source === undefined || (query.source === "file") === (type === "document")) &&
         (!query.folderIds?.length || query.withInternal === true || type === "document"));
-    const rows = chosen.flatMap((corpus) => searchCorpus(context, query, corpus));
+    const rows = chosen.flatMap((corpus) => searchCorpus(context, { ...query, ...(query.folderIds ? { accountIds: query.folderIds } : {}) }, corpus));
     const exactFirst = rows.some((row) => row.exact !== undefined);
     rows.sort((a, b) => query.newest
-        ? b.updatedAt - a.updatedAt || a.id - b.id
+        ? (b.at ?? 0) - (a.at ?? 0) || a.id - b.id
         : (exactFirst ? Number(b.exact === true) - Number(a.exact === true) : 0) ||
             Number(a.relevance === null) - Number(b.relevance === null) ||
             (a.relevance ?? 0) - (b.relevance ?? 0) ||
-            b.updatedAt - a.updatedAt ||
+            (b.at ?? 0) - (a.at ?? 0) ||
             a.id - b.id);
     const read = {
         document: context.database.prepare("SELECT * FROM documents WHERE id = ?"),
@@ -76,7 +82,7 @@ export const searchNotes = (context, query) => {
     };
     return {
         items: rows.slice(offset, offset + query.limit).map((row) => {
-            const found = read[row.type].get(row.id);
+            const found = read[row.type === "document" ? "document" : "note"].get(row.id);
             const note = row.type === "document" ? documentOf(found) : noteOf(found);
             return {
                 ref: note.ref,
@@ -88,7 +94,7 @@ export const searchNotes = (context, query) => {
         hasMore: rows.length > offset + query.limit,
     };
 };
-const searchCorpus = (context, query, corpus) => {
+export const searchCorpus = (context, query, corpus) => {
     const { database } = context;
     const wordMatch = (match) => ({
         sql: `n.id IN (SELECT rowid FROM ${corpus.words} WHERE ${corpus.words} MATCH ?)`,
@@ -116,8 +122,8 @@ const searchCorpus = (context, query, corpus) => {
         .map((row) => ({ id: Number(row.id), title: normalize(String(row.title)) })));
     const leaf = (node) => {
         const { field, operator, value } = node;
-        if (!NOTE_FIELDS.includes(field))
-            searchError("query.unsupported-field", node.span, `${field}: is not a note field — notes take ${NOTE_FIELDS.join(", ")}`);
+        if (!corpus.fields.includes(field))
+            searchError("query.unsupported-field", node.span, `${field}: is not ${corpus.noun === "event" ? "an" : "a"} ${corpus.noun} field — ${corpus.noun}s take ${corpus.fields.join(", ")}`);
         if ((field === "text" || field === "exact") && value === "")
             return bound("0");
         if (isStemmed(node)) {
@@ -168,7 +174,7 @@ const searchCorpus = (context, query, corpus) => {
             if (tag === undefined)
                 searchError("query.invalid-tag", node.span);
             // A label on a folder or subfolder labels every note under it, at any depth.
-            return corpus.tagged(canonicalTagNamed(database, tag)?.name ?? tag);
+            return corpus.tagged?.(canonicalTagNamed(database, tag)?.name ?? tag) ?? bound("0");
         }
         if (field === "source")
             return bound(value.toLowerCase() === (corpus.type === "note" ? "internal" : "file") ? "1" : "0");
@@ -199,15 +205,11 @@ const searchCorpus = (context, query, corpus) => {
             if (!range)
                 searchError("query.invalid-ast", node.span);
             return combine([
-                ...(range.lower === undefined
-                    ? []
-                    : [bound(`n.updated_at ${range.lower_inclusive ? ">=" : ">"} ?`, range.lower)]),
-                ...(range.upper === undefined
-                    ? []
-                    : [bound(`n.updated_at ${range.upper_inclusive ? "<=" : "<"} ?`, range.upper)]),
+                ...(range.lower === undefined ? [] : [bound(`n.at ${range.lower_inclusive ? ">=" : ">"} ?`, range.lower)]),
+                ...(range.upper === undefined ? [] : [bound(`n.at ${range.upper_inclusive ? "<=" : "<"} ?`, range.upper)]),
             ], "AND");
         }
-        return bound(["notes", "note"].includes(value.toLowerCase()) ? "1" : "0");
+        return bound([corpus.noun, `${corpus.noun}s`].includes(value.toLowerCase()) ? "1" : "0");
     };
     const compile = (node) => {
         if (node.kind === "predicate") {
@@ -243,10 +245,8 @@ const searchCorpus = (context, query, corpus) => {
         const stemmed = [...fragments.keys()].some(isStemmed);
         const scope = combine([
             bound("n.deleted_at IS NULL"),
-            ...(corpus.type === "document" ? [bound("coalesce(json_extract(n.metadata, '$.catalog.active'), 1)=1")] : []),
-            ...(query.folderIds?.length && corpus.type === "document"
-                ? [bound("n.folder_id IN (SELECT CAST(value AS INTEGER) FROM json_each(?))", JSON.stringify(query.folderIds))]
-                : []),
+            ...(corpus.live ? [bound(corpus.live)] : []),
+            ...(query.accountIds?.length && corpus.inAccounts ? [corpus.inAccounts(JSON.stringify(query.accountIds))] : []),
         ], "AND");
         const where = combine([scope, expression], "AND");
         const ranking = query.newest ? undefined : required(query.root, stemmed ? "stems" : "fts");
@@ -263,8 +263,8 @@ const searchCorpus = (context, query, corpus) => {
             cteParams.push(`normalized_text : (${exactTier})`);
         }
         const order = query.newest
-            ? "n.updated_at DESC, n.id"
-            : `${exactTier ? "exact DESC, " : ""}${ranking ? "f.rank IS NULL, f.rank, " : ""}n.updated_at DESC, n.id`;
+            ? "n.at DESC, n.id"
+            : `${exactTier ? "exact DESC, " : ""}${ranking ? "f.rank IS NULL, f.rank, " : ""}n.at DESC, n.id`;
         const offset = query.offset ?? 0;
         // With a JS test the SQL only names candidates: every leaf's SQL answer comes along, so the test sees it.
         const leaves = [...fragments.entries()].filter(([node]) => !tests.has(node));
@@ -304,7 +304,7 @@ const searchCorpus = (context, query, corpus) => {
         return matched.map((row) => ({
             type: corpus.type,
             id: Number(row.id),
-            updatedAt: Number(row.updated_at),
+            at: row.at == null ? null : Number(row.at),
             relevance: row.relevance === null || row.relevance === undefined ? null : Number(row.relevance),
             ...(exactTier ? { exact: Number(row.exact) === 1 } : {}),
         }));

package/dist/store/sqlite/notes.d.ts

@@ -2,13 +2,13 @@ import type { Section } from "../../readers/types.js";
 import type { CacheDatabase } from "../driver.js";
 import { type DocumentMetadata } from "./document-metadata.js";
 import { type ImportedDocuments } from "./imported-documents.js";
-import { LINK_KINDS } from "./link-kinds.js";
+import { LINK_KINDS, LINK_ORIGINS, type LINK_SOURCES } from "./link-kinds.js";
 import type { NoteSearch } from "./note-search.js";
 import type { StoreContext } from "./open.js";
 import { type Label } from "./tags.js";
 import { type Thing } from "./things.js";
 export declare const NOTE_FORMATS: readonly ["obsidian", "markdown"];
-export declare const LINK_ORIGINS: readonly ["file", "owner", "suggested", "message"];
+export { LINK_ORIGINS } from "./link-kinds.js";
 /**
  * A notes folder: an `accounts` row of provider `local` or `obsidian`. Where it is on disk is this computer's config. A
  * documents account (Drive's, one per Google account) is read and written the same way and names its `account`.
@@ -52,7 +52,7 @@ export interface Link {
     to: string | null;
     kind: (typeof LINK_KINDS)[number];
     anchor: string | null;
-    origin: (typeof LINK_ORIGINS)[number];
+    origin: (typeof LINK_SOURCES)[number];
     targetText: string | null;
     role: string | null;
     evidence: string | null;
@@ -76,7 +76,7 @@ export interface LinkInput {
     targetText?: string;
     kind: Link["kind"];
     anchor?: string;
-    origin?: Link["origin"];
+    origin?: (typeof LINK_ORIGINS)[number];
     role?: string;
     evidence?: string;
     provenance?: string;
@@ -193,4 +193,3 @@ export declare const resolvePersonLinks: (database: CacheDatabase, names: (strin
 /** Writes one link; an identical resolved link is answered instead of doubled. */
 export declare const insertLink: (context: StoreContext, input: LinkInput) => Link;
 export declare const notesStoreOver: (context: StoreContext) => Omit<NotesStore, keyof NoteSearch>;
-export {};

package/dist/store/sqlite/notes.js

@@ -10,13 +10,13 @@ import { ownerPerson } from "./actors.js";
 import { encodeDocumentMetadata, languageOf } from "./document-metadata.js";
 import { PREFERRED_FILE, syncFile } from "./files.js";
 import { importedDocumentWhere } from "./imported-documents.js";
-import { LINK_KINDS, linkKind } from "./link-kinds.js";
+import { LINK_KINDS, LINK_ORIGINS, linkKind } from "./link-kinds.js";
 import { reviewLink } from "./link-reviews.js";
 import { addTags, ensureTag, labelsOf } from "./tags.js";
 import { canonicalThingReference, referenceOfThing, storedThing, thingOf } from "./things.js";
 import { write } from "./transaction.js";
 export const NOTE_FORMATS = ["obsidian", "markdown"];
-export const LINK_ORIGINS = ["file", "owner", "suggested", "message"];
+export { LINK_ORIGINS } from "./link-kinds.js";
 const iso = (value) => new Date(Number(value)).toISOString();
 const optionalIso = (value) => (value == null ? null : iso(value));
 const textOf = (text, what, max = 100_000, allowEmpty = false) => {

package/dist/store/sqlite/project-transfer/snapshot.d.ts

@@ -23,7 +23,7 @@ export declare const projectAnnotationSnapshot: (database: CacheDatabase, projec
             ref: string;
             from: string;
             to: string | null;
-            kind: "about" | "answered-by" | "assigned-to" | "blocked-by" | "created-from" | "duplicate-of" | "evidence" | "follow-up-of" | "labelled" | "links-to" | "member-of" | "related-to";
+            kind: "about" | "answered-by" | "assigned-to" | "attached" | "blocked-by" | "created-from" | "duplicate-of" | "evidence" | "follow-up-of" | "labelled" | "links-to" | "member-of" | "related-to";
             origin: "file" | "message" | "owner" | "suggested";
             anchor: string | null;
             target_text: string | null;

package/dist/store/store.d.ts

@@ -20,6 +20,7 @@ import { type AgentActionsStore } from "./sqlite/agent-actions.js";
 import { type BotUpdateStore } from "./sqlite/bot-updates.js";
 import { type CatalogStore } from "./sqlite/catalog.js";
 import { type MailStore } from "./sqlite/emails.js";
+import { type EventSearch } from "./sqlite/event-search.js";
 import { type FactsStore } from "./sqlite/facts.js";
 import { type FilesStore } from "./sqlite/files.js";
 import { type InvolvementStore } from "./sqlite/involvements.js";
@@ -84,7 +85,7 @@ export interface MessageStore {
     /** Where each document is kept: its files, their folders and paths. */
     readonly files: FilesStore;
     /** Calendars and their entries, the source records of events: the `EventStore` a calendar CLI writes through. */
-    readonly calendar: EventStore;
+    readonly calendar: EventStore & EventSearch;
     /** Choices that hold until replaced, each with its evidence. */
     readonly meetingFindings: MeetingFindingsStore;
     /** Cited facts and follow-ups about one project from stored messages, mail and documents, for review. */

package/dist/store/store.js

@@ -19,6 +19,7 @@ import { eventStoreOver } from "./sqlite/calendar.js";
 import { catalogStoreOver } from "./sqlite/catalog.js";
 import { contextEvidenceOver, contextIndexOver, contextRecordsOver } from "./sqlite/context-evidence.js";
 import { mailStoreOver } from "./sqlite/emails.js";
+import { eventSearchOver } from "./sqlite/event-search.js";
 import { evidenceFindingsStoreOver } from "./sqlite/evidence-findings.js";
 import { factsStoreOver } from "./sqlite/facts.js";
 import { filesStoreOver } from "./sqlite/files.js";
@@ -100,7 +101,7 @@ const storeOver = (context) => {
     const notes = once(() => ({ ...notesStoreOver(context), ...noteSearchOver(context) }));
     const catalog = once(() => catalogStoreOver(context));
     const files = once(() => filesStoreOver(context));
-    const calendar = once(() => eventStoreOver(context));
+    const calendar = once(() => ({ ...eventStoreOver(context), ...eventSearchOver(context) }));
     const meetingFindings = once(() => meetingFindingsStoreOver(context));
     const evidenceFindings = once(() => evidenceFindingsStoreOver(context));
     const facts = once(() => factsStoreOver(context));