@agent-native/core 0.200.0-nightly-20261002124406 → 0.200.0

package/dist/search/query-parser.js

@@ -0,0 +1,71 @@
+export function parseSearchQuery(input) {
+    const groups = [];
+    const negatives = [];
+    let pendingOr = false;
+    let i = 0;
+    const pushTerm = (term) => {
+        if (!/[\p{L}\p{N}]/u.test(term.text)) {
+            pendingOr = false;
+            return;
+        }
+        if (term.negated) {
+            negatives.push(term);
+            pendingOr = false;
+            return;
+        }
+        if (pendingOr && groups.length > 0) {
+            groups[groups.length - 1].terms.push(term);
+        }
+        else {
+            groups.push({ terms: [term] });
+        }
+        pendingOr = false;
+    };
+    const trimmed = input.trim();
+    const length = trimmed.length;
+    while (i < length) {
+        if (/\s/.test(trimmed[i])) {
+            i += 1;
+            continue;
+        }
+        let negated = false;
+        if (trimmed[i] === "-" && trimmed[i + 1] && !/\s/.test(trimmed[i + 1])) {
+            negated = true;
+            i += 1;
+        }
+        let titleOnly = false;
+        if (trimmed.startsWith("intitle:", i)) {
+            titleOnly = true;
+            i += "intitle:".length;
+        }
+        if (i >= length)
+            break;
+        if (trimmed[i] === '"') {
+            const close = trimmed.indexOf('"', i + 1);
+            const end = close === -1 ? length : close;
+            const text = trimmed.slice(i + 1, end);
+            i = close === -1 ? length : end + 1;
+            if (text.trim())
+                pushTerm({ text: text.trim(), phrase: true, negated, titleOnly });
+            continue;
+        }
+        let j = i;
+        while (j < length && !/\s/.test(trimmed[j]))
+            j += 1;
+        const text = trimmed.slice(i, j);
+        i = j;
+        if (!negated && !titleOnly && text === "OR" && groups.length > 0) {
+            pendingOr = true;
+            continue;
+        }
+        pushTerm({ text, phrase: false, negated, titleOnly });
+    }
+    return {
+        groups,
+        negatives,
+        empty: groups.length === 0 && negatives.length === 0,
+    };
+}
+export function searchQueryNeedles(parsed) {
+    return parsed.groups.flatMap((group) => group.terms.map((term) => term.text));
+}

package/dist/search/query.d.ts

@@ -0,0 +1,46 @@
+/**
+ * SQL for searching through the core index, as Drizzle fragments an app
+ * composes into its own query. The app keeps its own access rule, filters,
+ * paging, and output; the index supplies matching and ranking.
+ *
+ * Matching, for each term:
+ * - titles and summaries match anywhere, including mid-word ("prio" finds
+ *   "Task Priorities"), because they're short;
+ * - bodies match whole words, every word as a prefix, through the GIN index;
+ *   mid-word body matches are deliberately not supported. A phrase needs
+ *   its words adjacent, except in a document too long or repetitive for
+ *   Postgres to keep every word position, where every word being in one
+ *   field is enough.
+ *
+ * Ranking reproduces the tiers the browser lane uses
+ * (`TITLE_MATCH_TIER` in Content): exact title 5, title prefix 4, title word
+ * prefixes 3, title substrings 2, title or summary 1, then how many query
+ * groups the title and summary cover, then whether the body contains the
+ * query as a phrase. Callers add their own tie-breaks.
+ *
+ * A query with a term longer than the index can match as a phrase throws
+ * `SearchTermTooLongError`; answer it with the app's fallback search.
+ */
+import { type SQL } from "drizzle-orm";
+import type { ParsedSearchQuery } from "./query-parser.js";
+import type { SearchableResourceRegistration } from "./registry.js";
+export interface IndexedSearchOptions {
+    registration: SearchableResourceRegistration;
+    query: Pick<ParsedSearchQuery, "groups" | "negatives">;
+    /** "title" matches titles only. Defaults to title, summary, and body. */
+    fields?: "all" | "title";
+}
+export interface IndexedSearchSql {
+    /** Join target and condition: `.innerJoin(search.join, search.on)`. */
+    join: SQL;
+    on: SQL;
+    /** Every group matches and no negative does. */
+    match: SQL;
+    matchTier: SQL<number>;
+    titleCoverage: SQL<number>;
+    summaryCoverage: SQL<number>;
+    bodyPhrase: SQL<number>;
+    /** The ranking terms above, best first, leaving out any that are always 0. */
+    orderBy: SQL[];
+}
+export declare function indexedSearchSql(options: IndexedSearchOptions): IndexedSearchSql;

package/dist/search/query.js

@@ -0,0 +1,149 @@
+/**
+ * SQL for searching through the core index, as Drizzle fragments an app
+ * composes into its own query. The app keeps its own access rule, filters,
+ * paging, and output; the index supplies matching and ranking.
+ *
+ * Matching, for each term:
+ * - titles and summaries match anywhere, including mid-word ("prio" finds
+ *   "Task Priorities"), because they're short;
+ * - bodies match whole words, every word as a prefix, through the GIN index;
+ *   mid-word body matches are deliberately not supported. A phrase needs
+ *   its words adjacent, except in a document too long or repetitive for
+ *   Postgres to keep every word position, where every word being in one
+ *   field is enough.
+ *
+ * Ranking reproduces the tiers the browser lane uses
+ * (`TITLE_MATCH_TIER` in Content): exact title 5, title prefix 4, title word
+ * prefixes 3, title substrings 2, title or summary 1, then how many query
+ * groups the title and summary cover, then whether the body contains the
+ * query as a phrase. Callers add their own tie-breaks.
+ *
+ * A query with a term longer than the index can match as a phrase throws
+ * `SearchTermTooLongError`; answer it with the app's fallback search.
+ */
+import { and, not, or, sql } from "drizzle-orm";
+import { SEARCH_RESOURCES_TABLE } from "./index-store.js";
+import { anyOfTsquery, isPhraseTerm, normalizeSearchText, termTsquery, } from "./tokenize.js";
+const ALIAS = "search_index";
+/** The indexer's weights for title, summary, and body. */
+const FIELD_WEIGHTS = ["A", "B", "C"];
+function escapeLike(value) {
+    return value.replace(/([\\%_])/g, "\\$1");
+}
+function escapeRegex(value) {
+    return value.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&");
+}
+function contains(column, needle) {
+    return sql `${column} LIKE ${`%${escapeLike(needle)}%`} ESCAPE '\\'`;
+}
+function startsWith(column, needle) {
+    return sql `${column} LIKE ${`${escapeLike(needle)}%`} ESCAPE '\\'`;
+}
+function wordPrefix(column, needle) {
+    const pattern = escapeRegex(needle).replace(/\s+/g, "[[:space:]]+");
+    return sql `${column} ~* ${`(^|[^[:alnum:]_])${pattern}`}`;
+}
+function sumOf(conditions) {
+    if (!conditions.length)
+        return null;
+    return sql `(${sql.join(conditions.map((condition) => sql `case when ${condition} then 1 else 0 end`), sql ` + `)})`;
+}
+export function indexedSearchSql(options) {
+    const { registration, query } = options;
+    // SQL is built per call, never at import: apps' tests stub drizzle-orm and
+    // still import core search.
+    const titleNorm = sql.raw(`${ALIAS}.title_norm`);
+    const summaryNorm = sql.raw(`${ALIAS}.summary_norm`);
+    // A bare constant in ORDER BY is read as a column position, so ranking
+    // terms that can only be zero are left out of the order rather than
+    // written as 0.
+    const zero = sql `0`;
+    const titleOnlyField = options.fields === "title";
+    const needle = (term) => normalizeSearchText(term.text);
+    const bodyTerms = (terms) => titleOnlyField ? [] : terms.filter((term) => !term.titleOnly);
+    const bodyMatch = (terms) => {
+        const tsquery = anyOfTsquery(terms.map((term) => termTsquery(term.text, { prefix: true })));
+        if (!tsquery)
+            return undefined;
+        // A document whose vector lost positions can't be phrase-matched
+        // exactly, so a phrase matches it when every word is in one field.
+        const anyOrder = anyOfTsquery(terms
+            .filter((term) => isPhraseTerm(term.text))
+            .flatMap((term) => FIELD_WEIGHTS.map((weights) => termTsquery(term.text, { prefix: true, anyOrder: true, weights }))));
+        const vectorMatch = anyOrder
+            ? sql `(doc_vector @@ ${tsquery}::tsquery OR (NOT positions_complete AND doc_vector @@ ${anyOrder}::tsquery))`
+            : sql `doc_vector @@ ${tsquery}::tsquery`;
+        return sql `${sql.raw(`${ALIAS}.resource_id`)} IN (SELECT resource_id FROM ${sql.raw(SEARCH_RESOURCES_TABLE)} WHERE app = ${registration.app} AND resource_type = ${registration.type} AND ${vectorMatch})`;
+    };
+    const titleAny = (terms) => or(...terms.map((term) => contains(titleNorm, needle(term))));
+    const summaryAny = (terms) => {
+        const eligible = bodyTerms(terms);
+        return eligible.length
+            ? or(...eligible.map((term) => contains(summaryNorm, needle(term))))
+            : undefined;
+    };
+    const anyField = (terms) => or(titleAny(terms), summaryAny(terms), bodyMatch(bodyTerms(terms)));
+    const groups = query.groups.filter((group) => group.terms.length > 0);
+    const match = groups.length || query.negatives.length
+        ? and(...groups.map((group) => anyField(group.terms)), ...query.negatives.map((negative) => not(anyField([negative]))))
+        : sql `false`;
+    const simpleQueries = groups.length > 0 && groups.every((group) => group.terms.length === 1)
+        ? [groups.map((group) => group.terms[0].text.trim()).join(" ")]
+        : groups.length === 1
+            ? groups[0].terms.map((term) => term.text.trim())
+            : [];
+    const normalizedSimpleQueries = simpleQueries
+        .map(normalizeSearchText)
+        .filter(Boolean);
+    const guardTitle = groups.every((group) => group.terms.every((term) => !/\s/.test(term.text)));
+    const allTitleSubstrings = groups.length
+        ? and(...groups.map((group) => titleAny(group.terms)))
+        : sql `false`;
+    const allTitleWordPrefixes = groups.length
+        ? and(...groups.map((group) => or(...group.terms.map((term) => wordPrefix(titleNorm, needle(term))))))
+        : sql `false`;
+    const allTitleOrSummary = groups.length
+        ? and(...groups.map((group) => or(titleAny(group.terms), summaryAny(group.terms))))
+        : sql `false`;
+    const simpleTier = (compare) => normalizedSimpleQueries.length
+        ? and(guardTitle ? allTitleSubstrings : sql `true`, or(...normalizedSimpleQueries.map(compare)))
+        : sql `false`;
+    const matchTier = sql `case
+    when ${simpleTier((simple) => sql `${titleNorm} = ${simple}`)} then 5
+    when ${simpleTier((simple) => startsWith(titleNorm, simple))} then 4
+    when ${allTitleSubstrings} and ${allTitleWordPrefixes} then 3
+    when ${allTitleSubstrings} then 2
+    when ${allTitleOrSummary} then 1
+    else 0
+  end`;
+    const titleCoverage = sumOf(groups.map((group) => titleAny(group.terms)));
+    const summaryCoverage = sumOf(groups.flatMap((group) => {
+        const condition = summaryAny(group.terms);
+        return condition ? [condition] : [];
+    }));
+    // The whole query as a phrase in the body, when it is two or more plain
+    // words, one per group.
+    const phraseTerms = !titleOnlyField &&
+        groups.length >= 2 &&
+        groups.every((group) => group.terms.length === 1 && !group.terms[0].titleOnly)
+        ? groups.map((group) => group.terms[0].text.trim()).join(" ")
+        : null;
+    const phraseQuery = phraseTerms
+        ? termTsquery(phraseTerms, { prefix: true, weights: "C" })
+        : null;
+    const bodyPhrase = phraseQuery
+        ? sql `case when ${sql.raw(`${ALIAS}.doc_vector`)} @@ ${phraseQuery}::tsquery then 1 else 0 end`
+        : null;
+    return {
+        join: sql `${sql.raw(SEARCH_RESOURCES_TABLE)} AS ${sql.raw(ALIAS)}`,
+        on: sql `${sql.raw(`${ALIAS}.app`)} = ${registration.app} AND ${sql.raw(`${ALIAS}.resource_type`)} = ${registration.type} AND ${sql.raw(`${ALIAS}.resource_id`)} = ${registration.idColumn}::text`,
+        match,
+        matchTier,
+        titleCoverage: titleCoverage ?? zero,
+        summaryCoverage: summaryCoverage ?? zero,
+        bodyPhrase: bodyPhrase ?? zero,
+        orderBy: [matchTier, titleCoverage, summaryCoverage, bodyPhrase]
+            .filter((term) => term !== null)
+            .map((term) => sql `${term} desc`),
+    };
+}

package/dist/search/registry.d.ts

@@ -0,0 +1,55 @@
+import { type AnyColumn, type Table } from "drizzle-orm";
+import { type MigrationEntry } from "../db/migrations.js";
+import { type ResourceChangeSource } from "../resource-changes/store.js";
+/** The change-feed consumer name the search index subscribes as. */
+export declare const SEARCH_CHANGE_CONSUMER = "search";
+/** What one resource contributes to the index. */
+export interface SearchableResourceDocument {
+    id: string;
+    title: string;
+    /** A short description, ranked between the title and the body. */
+    summary?: string | null;
+    body?: string | null;
+    modifiedAt?: string | Date | null;
+}
+export interface SearchableResourceRegistration {
+    /** The app that owns the table, e.g. "content". */
+    app: string;
+    /** Matches the shareable resource type, e.g. "document". */
+    type: string;
+    /** The Drizzle table whose rows are indexed. */
+    table: Table;
+    /** Its primary key column. */
+    idColumn: AnyColumn;
+    /**
+     * Bump when `load` changes what it returns. A higher version rebuilds the
+     * index; searches use the app's fallback until the rebuild completes.
+     */
+    version: number;
+    /**
+     * Loads the searchable text for a batch of ids. Omit an id whose row no
+     * longer exists and it is removed from the index. Index every row the app
+     * might search; access and filters are checked live at query time.
+     */
+    load(ids: string[]): Promise<SearchableResourceDocument[]>;
+}
+/**
+ * Makes an app table searchable through the core index. Pair it with
+ * `searchIndexMigration()` in the app's `runMigrations` list, which installs
+ * the triggers that keep the index fresh.
+ */
+export declare function registerSearchableResource(registration: SearchableResourceRegistration): SearchableResourceRegistration;
+export declare function getSearchableResource(app: string, type: string): SearchableResourceRegistration | undefined;
+export declare function listSearchableResources(): SearchableResourceRegistration[];
+/** Removes a registration. Tests use it to isolate cases. */
+export declare function unregisterSearchableResource(app: string, type: string): void;
+export declare function searchableResourceSource(registration: SearchableResourceRegistration): ResourceChangeSource;
+/**
+ * A named migration that creates the search tables and installs change
+ * capture on the registration's table. Add it to the app's `runMigrations`
+ * list with the app's next version number.
+ */
+export declare function searchIndexMigration(registration: SearchableResourceRegistration, entry: {
+    version: number;
+    name: string;
+}): MigrationEntry;

package/dist/search/registry.js

@@ -0,0 +1,93 @@
+import { getTableName } from "drizzle-orm";
+import { deferMigration } from "../db/migrations.js";
+import { registerRecurringSweepHandler } from "../jobs/sweep-hooks.js";
+import { assertResourceKey, installResourceChangeCapture, registerAfterWriteDrain, resourceChangeTriggerNames, } from "../resource-changes/store.js";
+import { ensureSearchIndexTables } from "./index-store.js";
+/** The change-feed consumer name the search index subscribes as. */
+export const SEARCH_CHANGE_CONSUMER = "search";
+const registrations = new Map();
+function key(app, type) {
+    return `${app}:${type}`;
+}
+/** Budget for the drain a write schedules in its own request. */
+const AFTER_WRITE_DRAIN_MS = 250;
+/** Share of the recurring sweep's budget search may use. */
+const SWEEP_DRAIN_MS = 20_000;
+let hooksRegistered = false;
+function registerDrainHooks() {
+    if (hooksRegistered)
+        return;
+    hooksRegistered = true;
+    // Both run only where the database is already awake: right after a write,
+    // and inside the per-minute sweep that already queries it. Search never
+    // polls on its own schedule.
+    registerAfterWriteDrain("search-index", async () => {
+        const { drainAllSearchIndexes } = await import("./indexer.js");
+        await drainAllSearchIndexes(Date.now() + AFTER_WRITE_DRAIN_MS);
+    });
+    registerRecurringSweepHandler("search-index", async ({ deadlineAt }) => {
+        const { drainAllSearchIndexes } = await import("./indexer.js");
+        await drainAllSearchIndexes(Math.min(deadlineAt - 1_000, Date.now() + SWEEP_DRAIN_MS));
+    });
+}
+/**
+ * Makes an app table searchable through the core index. Pair it with
+ * `searchIndexMigration()` in the app's `runMigrations` list, which installs
+ * the triggers that keep the index fresh.
+ */
+export function registerSearchableResource(registration) {
+    assertResourceKey(registration.app, "Search app");
+    assertResourceKey(registration.type, "Search resource type");
+    if (!Number.isInteger(registration.version) || registration.version < 1) {
+        throw new Error("Search registration version must be a positive integer.");
+    }
+    const triggers = resourceChangeTriggerNames(searchableResourceSource(registration)).function;
+    for (const other of registrations.values()) {
+        if (key(other.app, other.type) === key(registration.app, registration.type))
+            continue;
+        if (resourceChangeTriggerNames(searchableResourceSource(other)).function ===
+            triggers) {
+            throw new Error(`Search registrations ${other.app}/${other.type} and ${registration.app}/${registration.type} would share change-capture trigger names. Rename one resource type.`);
+        }
+    }
+    registrations.set(key(registration.app, registration.type), registration);
+    registerDrainHooks();
+    return registration;
+}
+export function getSearchableResource(app, type) {
+    return registrations.get(key(app, type));
+}
+export function listSearchableResources() {
+    return [...registrations.values()];
+}
+/** Removes a registration. Tests use it to isolate cases. */
+export function unregisterSearchableResource(app, type) {
+    registrations.delete(key(app, type));
+}
+export function searchableResourceSource(registration) {
+    return {
+        app: registration.app,
+        resourceType: registration.type,
+        table: getTableName(registration.table),
+        idColumn: registration.idColumn.name,
+    };
+}
+/**
+ * A named migration that creates the search tables and installs change
+ * capture on the registration's table. Add it to the app's `runMigrations`
+ * list with the app's next version number.
+ */
+export function searchIndexMigration(registration, entry) {
+    return {
+        version: entry.version,
+        name: entry.name,
+        sql: {},
+        run: async (exec) => {
+            await ensureSearchIndexTables(exec);
+            const installed = await installResourceChangeCapture(exec, searchableResourceSource(registration), SEARCH_CHANGE_CONSUMER);
+            // A busy table: try again on the next boot rather than block writes.
+            if (!installed)
+                return deferMigration();
+        },
+    };
+}

package/dist/search/tokenize.d.ts

@@ -0,0 +1,103 @@
+/**
+ * Search tokens, computed in JavaScript for both the index and the query.
+ *
+ * Postgres's text parser depends on the database's locale: on some locales it
+ * drops Japanese text entirely, and PGlite and Neon don't agree. So core never
+ * asks Postgres to tokenize. It builds `tsvector` and `tsquery` literals
+ * itself, and the database only stores, indexes, and matches them. The same
+ * code tokenizes documents and queries, so they always agree.
+ *
+ * Rules:
+ * - Text is NFKC-normalized and lowercased. There is no stemming and there are
+ *   no stopwords.
+ * - A word is a run of letters, numbers, and combining marks. Everything else
+ *   separates words, so `snake_case`, `kebab-case`, URLs, and paths become
+ *   their parts at consecutive positions, and a query for the same text
+ *   matches them as a phrase.
+ * - A camelCase or PascalCase word is indexed whole and as its parts, so
+ *   "camelCase", "camel", and "case camel" all find it.
+ * - A word longer than Postgres's 2,046-byte limit keeps its longest prefix
+ *   that fits.
+ * - Chinese, Japanese, and Korean runs become overlapping character pairs,
+ *   because they have no spaces to split on. A query becomes the same pairs
+ *   as a phrase, which matches exactly that substring. A run's last character
+ *   starts no pair, so it is also indexed alone, and a one-character query
+ *   finds every character as a prefix.
+ */
+export type SearchWeight = "A" | "B" | "C" | "D";
+/** Normalized form used for title and summary comparisons. */
+export declare function normalizeSearchText(value: string): string;
+/** UTF-8 length without encoding: 1-3 bytes per UTF-16 unit, 4 per pair. */
+export declare function utf8ByteLength(value: string): number;
+export interface SearchToken {
+    lexeme: string;
+    position: number;
+}
+/**
+ * Document tokens with positions starting at `start`. Returns the next free
+ * position so several fields can share one position space.
+ */
+export declare function documentTokens(text: string, start?: number): {
+    tokens: SearchToken[];
+    next: number;
+};
+/**
+ * Query lexemes for one term, in order. Unlike documents, a query word is
+ * never split on case: "camelCase" looks for the whole word, which documents
+ * index alongside its parts.
+ */
+export declare function queryLexemes(text: string): string[];
+export interface WeightedField {
+    text: string | null | undefined;
+    weight: SearchWeight;
+}
+export interface SearchVector {
+    /** A `tsvector` literal. */
+    literal: string;
+    /**
+     * False when Postgres's limits made the vector drop or merge word
+     * positions, so phrase matching can't be exact for this document.
+     */
+    positionsComplete: boolean;
+}
+/**
+ * A `tsvector` for the fields, in order, sharing one position space with a
+ * gap between fields, so a phrase never spans two of them. Postgres keeps at
+ * most 255 positions per word and none past 16,383. Past the last position,
+ * each field's words collapse onto its own position, because Postgres merges
+ * equal positions and keeps only the higher weight. A word over 255
+ * positions keeps its first position in each field. A very large document
+ * keeps one position per word in each field, and one with more distinct
+ * words than fit keeps the words that come first, so the vector stays under
+ * Postgres's size limit. Any of these makes `positionsComplete` false; each
+ * keeps which fields hold each word.
+ */
+export declare function buildSearchVector(fields: readonly WeightedField[]): SearchVector;
+/**
+ * A term with more words than the index can match. The query can't be
+ * answered from the index; the app's fallback search can answer it.
+ */
+export declare class SearchTermTooLongError extends RangeError {
+    constructor();
+}
+export interface QueryPhraseOptions {
+    /** Treat the last lexeme as a prefix. */
+    prefix?: boolean;
+    /** Restrict every lexeme to these weights. */
+    weights?: string;
+    /**
+     * Match the lexemes anywhere, in any order, instead of as a phrase. For
+     * documents whose positions aren't complete.
+     */
+    anyOrder?: boolean;
+}
+/**
+ * One term as a `tsquery` literal: a single lexeme, or a phrase of adjacent
+ * lexemes. Returns null when the term has nothing to match. Throws
+ * `SearchTermTooLongError` past `MAX_QUERY_LEXEMES`, which Postgres can't
+ * evaluate.
+ */
+export declare function termTsquery(text: string, options?: QueryPhraseOptions): string | null;
+/** Whether a term is more than one lexeme, so it matches as a phrase. */
+export declare function isPhraseTerm(text: string): boolean;
+export declare function anyOfTsquery(parts: readonly (string | null)[]): string | null;