datanotes 2.0.1 → 3.0.0

package/dist/api/docs.d.ts

@@ -0,0 +1,211 @@
+/**
+ * The documentation of this version of Datanotes, read offline: every page of datanotes.dev as
+ * Markdown, shipped in the npm package as `dist/docs/` (scripts/copy-docs.mjs copies them there
+ * after the docs build). The MCP server `datanotes mcp` serves it to agents as three tools
+ * (`list_docs`, `read_docs`, `search_docs`: src/api/mcp.ts), the engine to programs as
+ * `GET /datanotes/manual/<page>.md` (src/server/http-server.ts). They are not engine operations:
+ * nothing is journaled and no engine is needed.
+ *
+ * A page is named by its path on the site without `.md` (`guides/read-and-write/read-and-filter`), a section by the
+ * anchor of its heading (`guides/read-and-write/read-and-filter#compare-values-with-operators`), as on the site; the
+ * site's own addresses (`https://datanotes.dev/guides/read-and-write/read-and-filter/#…`) are read too. Every mistake
+ * says what is wrong, the closest real page or section, and how to list them (the hint standard
+ * of src/api/guidance.ts).
+ */
+import { type Guidance } from "./guidance.js";
+/** An environment variable that names the folder of the docs instead of the package's own. */
+export declare const DOCS_DIR_ENV = "DATANOTES_DOCS_DIR";
+/** The most characters read_docs returns at once: about 5 000 tokens. */
+export declare const READ_DOCS_MAX = 20000;
+/** The page for agents on reading the docs offline. */
+export declare const OFFLINE_DOCS = "https://datanotes.dev/guides/agents/docs-offline.md#the-agent-reads-the-docs-of-its-version";
+/** A mistake in reading the docs: what is wrong, how to fix it, where to read more. */
+export declare class DocsError extends Error {
+    readonly hint: string;
+    readonly docs: string;
+    /** `missing`: the docs are not in this installation; `not_found`: no such page or section; `bad_input`: wrong arguments. */
+    readonly code: "missing" | "not_found" | "bad_input";
+    constructor(message: string, g: Guidance, code: DocsError["code"]);
+}
+/** A page of the docs. */
+export interface DocsPage {
+    /** Its path on the site, without `.md`: `guides/read-and-write/read-and-filter` (the home page is `index`). */
+    path: string;
+    title: string;
+    description: string;
+}
+/** How a section matched a search: by its words (BM25), or by its meaning (the engine's vector index of the docs). */
+export type DocsSignal = "keyword" | "semantic";
+/** A section of a page that matches a search. */
+export interface DocsHit {
+    /** What read_docs reads: `guides/read-and-write/read-and-filter#compare-values-with-operators` (the page alone for its introduction). */
+    path: string;
+    page: string;
+    section: string;
+    excerpt: string;
+    /** BM25 for a keyword search; the Reciprocal Rank Fusion score for a hybrid one. */
+    score: number;
+    /** The rankings the section is in: `keyword` alone without semantic search. */
+    signals: DocsSignal[];
+}
+/** A section as search_docs ranks it, with the text the engine embeds for semantic search. */
+export interface DocsSection {
+    /** What read_docs reads, as in {@link DocsHit}. */
+    path: string;
+    page: string;
+    section: string;
+    /** The page's title and the heading, then the section's own prose, then its code. */
+    text: string;
+}
+interface Section {
+    anchor: string;
+    title: string;
+    level: number;
+    /** Where its heading starts, where it ends with its subsections, and where its own text ends. */
+    start: number;
+    end: number;
+    ownEnd: number;
+}
+interface Page extends DocsPage {
+    text: string;
+    sections: Section[];
+}
+/**
+ * The folder of the docs: `DATANOTES_DOCS_DIR` when set, else the package's `dist/docs` (next to
+ * `dist/datanotes.cjs`, above `dist/api/`, or in a checkout where the docs were built). Null when
+ * there is none.
+ */
+export declare function findDocsDir(env?: Record<string, string | undefined>, from?: string): string | null;
+/** A heading's anchor as the site makes it (github-slugger): lowercase, punctuation dropped, spaces as dashes. */
+export declare function slug(heading: string): string;
+/** A page or section as given (`guides/read-and-write/read-and-filter.md`, `/guides/read-and-write/read-and-filter/#x`, a datanotes.dev address): the page's path and the anchor. */
+export declare function docsRef(input: string): {
+    page: string;
+    anchor: string | null;
+};
+/** Common English words left out of queries and texts: they say nothing of the topic. */
+export declare const STOP_WORDS: ReadonlySet<string>;
+/**
+ * The stem of a lowercase word: plurals and verb endings stripped, so that the forms of a word meet
+ * (`tables`/`table`, `queries`/`query`, `renamed`/`renaming`/`rename`, `dropped`/`drop`,
+ * `rated`/`rate`, `using`/`use`). Not a dictionary form (`create` is `creat`), only the same for
+ * every form. Words of three letters or less, and words with digits, stay as they are.
+ */
+export declare function stem(word: string): string;
+/** The search terms of a text: folded words, without one-letter and common words, stemmed. */
+export declare function tokenize(text: string): string[];
+/**
+ * Words that mean the same thing in these docs, for queries: a word of the query also matches the
+ * others of its group, with SYNONYM_WEIGHT. An entry can be a phrase ("roll back"), which matches
+ * where its words follow each other, in the query and in the docs.
+ */
+export declare const SEARCH_SYNONYMS: readonly (readonly string[])[];
+/** How much a synonym counts against the word itself. */
+export declare const SYNONYM_WEIGHT = 0.7;
+/** A term of a query: the word (or synonym phrase) as stems, and every way it matches, itself first. */
+export interface QueryTerm {
+    term: string;
+    alternatives: {
+        stems: string[];
+        weight: number;
+    }[];
+}
+/**
+ * The terms of a query, each with its synonyms. A synonym phrase written in the query ("roll
+ * back", "time travel") is one term; a term repeated in the query counts once.
+ */
+export declare function queryTerms(query: string): QueryTerm[];
+/** A query of search_docs as written: the words to rank by, and the words a section must hold or must not. */
+export interface DocsQuery {
+    /** The words to rank by, the syntax taken out: `"time travel" +view -sql` ranks by `time travel view`. */
+    text: string;
+    /** Quoted phrases and +terms, as stems: each must appear in a section (its words in a row, in any form). */
+    required: string[][];
+    /** -terms and -"phrases", as stems: a section that holds one is left out. */
+    excluded: string[][];
+}
+/**
+ * Read the syntax of a search_docs query: `"a phrase"` and `+term` must appear in a section,
+ * `-term` (or `-"a phrase"`) must not, every other word ranks only. Curly quotes count as straight
+ * ones, a quote left open runs to the end, and a word that starts with two dashes (`--vault`) is a
+ * word. Words that say nothing (`+the`) require nothing.
+ */
+export declare function parseQuery(query: string): DocsQuery;
+/** The docs in a folder (null: there are none, and every call says how to get them). */
+export declare class DocsLibrary {
+    readonly dir: string | null;
+    private loaded;
+    private index;
+    private hash;
+    private sectionList;
+    constructor(dir: string | null);
+    private missing;
+    private all;
+    /** The pages, sorted by path; only those under `prefix` (`guides/`) when it is given. */
+    list(prefix?: string): DocsPage[];
+    private page;
+    private section;
+    /**
+     * The Markdown of a page (`guides/read-and-write/read-and-filter`) or of one section (`guides/read-and-write/read-and-filter#anchor`, or
+     * `section`), with its subsections. Without a cap, the whole text.
+     */
+    text(ref: string, section?: string | null): {
+        ref: string;
+        text: string;
+        page: Page;
+        from: number;
+    };
+    /**
+     * What read_docs returns: the page or section from `offset`, at most `max` characters, cut at the
+     * end of a line, then a note on how to read the rest (the sections that follow, or the offset).
+     */
+    read(ref: string, opts?: {
+        section?: string | null;
+        offset?: number;
+        max?: number;
+    }): string;
+    private searchIndex;
+    /** The terms of a query to rank by; a query with none is refused. */
+    private termsOf;
+    /**
+     * The units ranked by BM25 for the query's words, best first, without those the required and
+     * excluded words leave out. Each term of the query (a word in any form, or one of its synonyms)
+     * scores by BM25 in a section's text, and more in its heading and its page's title; rare terms
+     * count more than common ones; a heading with every term, and terms next to each other as in
+     * the query, add to it; a section with every term ranks above one with some.
+     */
+    private ranked;
+    /**
+     * The sections that best match `query`, best first, by their words (BM25: {@link ranked}). A
+     * quoted phrase or a +term must appear in a section, a -term must not (parseQuery).
+     */
+    search(query: string, limit?: number): DocsHit[];
+    /**
+     * Hybrid search: the ranking of {@link search} by words, and `semantic`, the sections ranked by
+     * meaning (the engine's vector index of these docs, best first), fused by Reciprocal Rank
+     * Fusion: a section scores the sum of 1 / (RRF_K + its rank) over the rankings it is in, among
+     * the first FUSION_DEPTH of each. The required and excluded words filter both rankings, and a
+     * section these docs do not have is left out. Each hit says which rankings it is in (`signals`).
+     */
+    hybrid(query: string, semantic: readonly {
+        path: string;
+    }[], limit?: number): DocsHit[];
+    /**
+     * Every section as search_docs ranks them (a page's introduction too), with the text to embed
+     * for semantic search: the page's title and the heading, the prose, then the code.
+     */
+    sections(): readonly DocsSection[];
+    /**
+     * The version of these docs: a hash of their sections (paths and texts). The engine's vector
+     * index of the docs is rebuilt when it changes, and search_docs fuses its ranking only when the
+     * engine's docs have the same version as its own.
+     */
+    version(): string;
+}
+/** The constant of Reciprocal Rank Fusion: 60, the value of its paper (Cormack et al., 2009) and of most search engines; it keeps the first ranks from outweighing agreement between the rankings. */
+export declare const RRF_K = 60;
+/** How many sections of each ranking hybrid search fuses. */
+export declare const FUSION_DEPTH = 50;
+/** The docs of a folder (default: `findDocsDir()`), loaded once per folder. */
+export declare function openDocs(dir?: string | null): DocsLibrary;
+export {};

package/dist/api/guidance.d.cts

@@ -0,0 +1,81 @@
+/**
+ * What an agent reads when a call goes wrong, on the client's side: the engine sends `hint` (how
+ * to fix it) and `docs` (where to read more) next to `error` (src/core/hints.ts is the engine's
+ * half), and the text of a failed call ends with them. Here also the mistakes caught before a call
+ * reaches the engine: a tool that does not exist, arguments the tool does not take, a required one
+ * missing, a value of the wrong type. Each names the closest real name and the tool's arguments.
+ *
+ * No dependency: src/api imports nothing else (the engine's copy of the docs address and of the
+ * name matching is in src/core/hints.ts; tests/hints.test.ts checks that both agree).
+ */
+/** The docs site; the pages are read in Markdown: `<route>.md`. */
+export declare const DOCS_BASE = "https://datanotes.dev";
+/** The page for agents on reading the engine's errors. */
+export declare const ERRORS_DOCS = "https://datanotes.dev/guides/agents/reading-errors.md";
+/** The overview of the operations, by kind. */
+export declare const OPERATIONS_DOCS = "https://datanotes.dev/reference/operations.md";
+/** The reference of an operation (its arguments, with types and defaults): the page of its kind; the overview for an unknown one. */
+export declare function toolDocs(op: string): string;
+/** How to fix a mistake, and where to read more. */
+export interface Guidance {
+    hint: string;
+    docs: string;
+}
+/** `hint` and `docs` of an engine's answer, or null when it has none. */
+export declare function guidanceIn(data: unknown): Guidance | null;
+/**
+ * A docs address of the site as the call that reads the same page in the docs shipped with the
+ * package (the read_docs tool of `datanotes mcp`, src/api/docs.ts):
+ * `read_docs("guides/read-and-write/read-and-filter#compare-values-with-operators")` for
+ * `https://datanotes.dev/guides/read-and-write/read-and-filter.md#compare-values-with-operators`. Null for any other address.
+ */
+export declare function localDocs(url: string): string | null;
+/**
+ * The text an agent reads: the message, then the hint and the docs page, one per line. The docs
+ * line also names the call that reads the page offline, when it is a page of the site.
+ */
+export declare function guidedText(message: string, g: Guidance | null): string;
+/**
+ * The text of an error for an agent of the MCP server, which always has the docs tools: each docs
+ * line keeps only the call that reads the page offline (`Docs: read_docs("…")`), with no address
+ * to fetch. A docs line that is not a page of the site keeps its address.
+ */
+export declare function offlineDocsText(text: string): string;
+/** The real names closest to a wrong one, best first (at most `max`). */
+export declare function closestNames(input: string, candidates: Iterable<string>, max?: number): string[];
+/** `Did you mean "x"?` / `Did you mean "x" or "y"?`, or "" when nothing is close. */
+export declare function didYouMean(input: string, candidates: Iterable<string>): string;
+type Schema = {
+    type?: string | string[];
+    anyOf?: Schema[];
+    items?: Schema;
+    enum?: unknown[];
+};
+/** A tool's input schema: its arguments and the required ones. */
+export type InputSchema = {
+    properties?: Record<string, Schema>;
+    required?: string[];
+};
+/** A schema's type in words: `string`, `object`, `list of strings`, `integer or null`, `"a" or "b"`. */
+export declare function typeName(s: Schema): string;
+/** An operation's arguments in one line: `table (string, required), values (object, required), dry_run (boolean)`. */
+export declare function argumentsOf(op: string): string;
+/** A tool's arguments in one line, from its input schema. */
+export declare function argumentList(schema: InputSchema): string;
+/** Whether a given value (not null) is of a schema's type and among its `enum`; a list's items too. A schema without a type takes anything. */
+export declare function typeOk(v: unknown, s: Schema): boolean;
+/** An error found before the call reaches the engine: its text names what to fix. */
+export interface ArgumentProblem extends Guidance {
+    message: string;
+}
+/** A tool that does not exist: the closest real ones. `names`: the tools the agent may call. */
+export declare function unknownTool(name: string, names: readonly string[]): ArgumentProblem;
+/**
+ * The arguments an agent gave a tool, against the tool's input schema: names it does not take (with
+ * the closest real one), required ones missing, values of the wrong type (null is "not given", as
+ * the engine reads it). Null when they are right; unknown tools are not checked here.
+ */
+export declare function checkArguments(op: string, args: Record<string, unknown>): ArgumentProblem | null;
+/** The same check for any tool, from its input schema (the docs tools of src/api/docs.ts), with its docs page. */
+export declare function checkToolArguments(op: string, schema: InputSchema, args: Record<string, unknown>, docs: string): ArgumentProblem | null;
+export {};

package/dist/api/guidance.d.ts

@@ -0,0 +1,81 @@
+/**
+ * What an agent reads when a call goes wrong, on the client's side: the engine sends `hint` (how
+ * to fix it) and `docs` (where to read more) next to `error` (src/core/hints.ts is the engine's
+ * half), and the text of a failed call ends with them. Here also the mistakes caught before a call
+ * reaches the engine: a tool that does not exist, arguments the tool does not take, a required one
+ * missing, a value of the wrong type. Each names the closest real name and the tool's arguments.
+ *
+ * No dependency: src/api imports nothing else (the engine's copy of the docs address and of the
+ * name matching is in src/core/hints.ts; tests/hints.test.ts checks that both agree).
+ */
+/** The docs site; the pages are read in Markdown: `<route>.md`. */
+export declare const DOCS_BASE = "https://datanotes.dev";
+/** The page for agents on reading the engine's errors. */
+export declare const ERRORS_DOCS = "https://datanotes.dev/guides/agents/reading-errors.md";
+/** The overview of the operations, by kind. */
+export declare const OPERATIONS_DOCS = "https://datanotes.dev/reference/operations.md";
+/** The reference of an operation (its arguments, with types and defaults): the page of its kind; the overview for an unknown one. */
+export declare function toolDocs(op: string): string;
+/** How to fix a mistake, and where to read more. */
+export interface Guidance {
+    hint: string;
+    docs: string;
+}
+/** `hint` and `docs` of an engine's answer, or null when it has none. */
+export declare function guidanceIn(data: unknown): Guidance | null;
+/**
+ * A docs address of the site as the call that reads the same page in the docs shipped with the
+ * package (the read_docs tool of `datanotes mcp`, src/api/docs.ts):
+ * `read_docs("guides/read-and-write/read-and-filter#compare-values-with-operators")` for
+ * `https://datanotes.dev/guides/read-and-write/read-and-filter.md#compare-values-with-operators`. Null for any other address.
+ */
+export declare function localDocs(url: string): string | null;
+/**
+ * The text an agent reads: the message, then the hint and the docs page, one per line. The docs
+ * line also names the call that reads the page offline, when it is a page of the site.
+ */
+export declare function guidedText(message: string, g: Guidance | null): string;
+/**
+ * The text of an error for an agent of the MCP server, which always has the docs tools: each docs
+ * line keeps only the call that reads the page offline (`Docs: read_docs("…")`), with no address
+ * to fetch. A docs line that is not a page of the site keeps its address.
+ */
+export declare function offlineDocsText(text: string): string;
+/** The real names closest to a wrong one, best first (at most `max`). */
+export declare function closestNames(input: string, candidates: Iterable<string>, max?: number): string[];
+/** `Did you mean "x"?` / `Did you mean "x" or "y"?`, or "" when nothing is close. */
+export declare function didYouMean(input: string, candidates: Iterable<string>): string;
+type Schema = {
+    type?: string | string[];
+    anyOf?: Schema[];
+    items?: Schema;
+    enum?: unknown[];
+};
+/** A tool's input schema: its arguments and the required ones. */
+export type InputSchema = {
+    properties?: Record<string, Schema>;
+    required?: string[];
+};
+/** A schema's type in words: `string`, `object`, `list of strings`, `integer or null`, `"a" or "b"`. */
+export declare function typeName(s: Schema): string;
+/** An operation's arguments in one line: `table (string, required), values (object, required), dry_run (boolean)`. */
+export declare function argumentsOf(op: string): string;
+/** A tool's arguments in one line, from its input schema. */
+export declare function argumentList(schema: InputSchema): string;
+/** Whether a given value (not null) is of a schema's type and among its `enum`; a list's items too. A schema without a type takes anything. */
+export declare function typeOk(v: unknown, s: Schema): boolean;
+/** An error found before the call reaches the engine: its text names what to fix. */
+export interface ArgumentProblem extends Guidance {
+    message: string;
+}
+/** A tool that does not exist: the closest real ones. `names`: the tools the agent may call. */
+export declare function unknownTool(name: string, names: readonly string[]): ArgumentProblem;
+/**
+ * The arguments an agent gave a tool, against the tool's input schema: names it does not take (with
+ * the closest real one), required ones missing, values of the wrong type (null is "not given", as
+ * the engine reads it). Null when they are right; unknown tools are not checked here.
+ */
+export declare function checkArguments(op: string, args: Record<string, unknown>): ArgumentProblem | null;
+/** The same check for any tool, from its input schema (the docs tools of src/api/docs.ts), with its docs page. */
+export declare function checkToolArguments(op: string, schema: InputSchema, args: Record<string, unknown>, docs: string): ArgumentProblem | null;
+export {};