@g3nomaly/analyze 0.0.0-stage → 0.1.0

package/dist/embeddings/abbreviations.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Replaces a short form from code with the word it stands for.
+ *
+ * @param word - A lowercase word.
+ * @returns The full word, or the word itself when it is no known short form.
+ */
+export declare function expandAbbreviation(word: string): string;

package/dist/embeddings/index-builder.d.ts

@@ -0,0 +1,10 @@
+import { type GraphNode, type SemanticIndex } from '../model';
+import type { EmbeddingModel, EmbeddingTable } from './model';
+/**
+ * Embeds every searchable node of a scan and packs the result, with the model
+ * itself, into the form a reader loads.
+ *
+ * @param table - The data of the same model. It travels with the bundle, so a reader can embed a query.
+ * @returns The index for `DocsBundle.semantic`.
+ */
+export declare function buildSemanticIndex(nodes: readonly GraphNode[], model: EmbeddingModel, table: EmbeddingTable): SemanticIndex;

package/dist/embeddings/index.d.ts

@@ -0,0 +1,10 @@
+export { expandAbbreviation } from './abbreviations';
+export { buildSemanticIndex } from './index-builder';
+export { EmbeddingModel, type EmbeddingTable, type WeightedText } from './model';
+export { type SemanticHit, SemanticSearcher } from './searcher';
+export { decodeTable, encodeTable, fromBase64, normalize, packNibbles, rowOf, similarity, toBase64, toBytes, unpackNibbles, } from './vectors';
+export { isPunctuation, WordVocabulary, wordsOf } from './words';
+/** Name of the embedding model the package ships, as the page shows it. */
+export declare const EMBEDDING_MODEL_NAME = "all-MiniLM-L6-v2 word table, 128 dimensions at 4 bits";
+/** Length of the vectors of the shipped model. */
+export declare const EMBEDDING_DIMENSIONS = 128;

package/dist/embeddings/model.d.ts

@@ -0,0 +1,113 @@
+/** An embedding model as data: one vector per word. */
+export interface EmbeddingTable {
+    /** Name of the model, for display. */
+    readonly name: string;
+    /** Length of every vector. */
+    readonly dimensions: number;
+    /** The words of the model. The index of a word is its row in `vectors`. */
+    readonly vocabulary: readonly string[];
+    /** One row of `dimensions` small whole numbers per word. The shipped model uses -7 to 7. */
+    readonly vectors: Int8Array;
+}
+/** One part of a text and how much it counts. */
+export interface WeightedText {
+    readonly text: string;
+    readonly weight: number;
+}
+/**
+ * Turns text into an embedding with a table of word vectors. The vector of a
+ * word was computed in advance by a contextual model that read the word on
+ * its own. The vector of a text is the mean of the vectors of its words.
+ * There is no neural network to run, so it works on any CPU and in a browser.
+ *
+ * Holds the table's vectors and a `WordVocabulary` over its words.
+ * `api/embedding.ts` creates one from the shipped table and passes it to
+ * `buildSemanticIndex`, which embeds every searchable node of a scan.
+ * `SemanticSearcher` creates one from the table stored in a bundle to embed
+ * what a reader types.
+ *
+ * @example
+ * const model = new EmbeddingModel({
+ *   name: index.model,
+ *   dimensions: index.dimensions,
+ *   vocabulary,
+ *   vectors: decodeTable(index.vocabularyVectors, vocabulary.length * index.dimensions),
+ * });
+ */
+export declare class EmbeddingModel {
+    readonly name: string;
+    readonly dimensions: number;
+    private readonly vectors;
+    private readonly vocabulary;
+    /** Median length of a row of the table. Computed on first use. */
+    private medianLength;
+    /**
+     * Creates a model from its table.
+     *
+     * @throws {Error} When the table does not hold one row per word.
+     */
+    constructor(table: EmbeddingTable);
+    /**
+     * Embeds one text.
+     *
+     * @returns A vector of length 1, or `null` when the model knows no word of the text.
+     */
+    embed(text: string): Float32Array | null;
+    /**
+     * Embeds a text made of parts that count differently, such as a name and a
+     * long description. Each part is embedded on its own, so a short part is
+     * not drowned out by a long one.
+     *
+     * @returns A vector of length 1, or `null` when the model knows no word of any part.
+     */
+    embedParts(parts: readonly WeightedText[]): Float32Array | null;
+    /**
+     * Lists the words of a text that carry its meaning: lowercase, identifiers
+     * split, short forms from code written out, without punctuation, single
+     * characters and function words such as `the`.
+     *
+     * @param text - Any text: prose, a name from code, a search query.
+     * @returns The words, each once, in text order.
+     */
+    contentWords(text: string): string[];
+    /**
+     * Looks up the vector of one word.
+     *
+     * @param word - A word from `contentWords`.
+     * @returns A vector of length 1, or `null` for a word the model does not have.
+     */
+    wordVector(word: string): Float32Array | null;
+    /**
+     * Says how much a word counts in a query. The table gives a word that says
+     * a lot a long row, so the length of the row is the weight.
+     *
+     * @param word - A word from `contentWords`.
+     * @returns The length of the word's row, or the median length for a word the model does not have.
+     */
+    wordWeight(word: string): number;
+    /**
+     * Says whether a word is one the table treats as saying little.
+     *
+     * @param word - A lowercase word.
+     * @returns `true` for a word of the model with a short row.
+     */
+    private isFunctionWord;
+    /**
+     * Reads one row of the table.
+     *
+     * @returns The values of the row. A view, not a copy.
+     */
+    private row;
+    /**
+     * Measures one row of the table.
+     *
+     * @returns The Euclidean length of the row.
+     */
+    private lengthOf;
+    /**
+     * Finds the median row length of the table.
+     *
+     * @returns The median, computed once.
+     */
+    private median;
+}

package/dist/embeddings/searcher.d.ts

@@ -0,0 +1,67 @@
+import type { NodeRef, SemanticIndex } from '../model';
+/** A node and how close its meaning is to the query. */
+export interface SemanticHit {
+    readonly node: NodeRef;
+    /** From 0 to 1: 1 is the same meaning, 0 is unrelated. */
+    readonly score: number;
+}
+/**
+ * Finds nodes by meaning in the index of a bundle.
+ *
+ * Holds the node vectors and word lists of a `SemanticIndex`, and decodes the
+ * `EmbeddingModel` stored in it on the first query. `searcherOf` in
+ * `query/semantic.ts` creates one per graph on its first search by meaning,
+ * for the page and the CLI alike; `scripts/search-benchmark.ts` creates one
+ * too. `search` scores every node against a query. `similarTo` compares one
+ * node with the others.
+ */
+export declare class SemanticSearcher {
+    private readonly index;
+    readonly modelName: string;
+    private readonly dimensions;
+    private readonly nodes;
+    private readonly nodeVectors;
+    private readonly rows;
+    private readonly words;
+    private readonly wordIds;
+    /** Decoded on the first query. The model is a few megabytes of text. */
+    private model;
+    /** The vector of each word of the index, `null` for a word the model does not have. Filled on the first query. */
+    private wordVectors;
+    /** Creates a searcher for one index. */
+    constructor(index: SemanticIndex);
+    /**
+     * Finds the nodes whose meaning is closest to a query.
+     *
+     * The score of a node has two parts. One compares the query as a whole
+     * with the name and the text of the node as a whole. The other takes each
+     * word of the query and finds the word of the node that is most like it:
+     * the same word counts 1, a word of similar meaning counts its likeness
+     * in the model. A one-word query is scored mostly by the second part.
+     *
+     * @param query - What the reader typed.
+     * @param minScore - Hits below this score are left out.
+     * @returns The hits, best first.
+     */
+    search(query: string, limit: number, minScore: number): SemanticHit[];
+    /**
+     * Finds the nodes whose meaning is closest to that of a node.
+     *
+     * @param minScore - Hits below this score are left out.
+     * @returns The hits, best first, without the node itself. Empty when the node has no vector.
+     */
+    similarTo(node: NodeRef, limit: number, minScore: number): SemanticHit[];
+    /**
+     * Decodes the model that came with the index.
+     *
+     * @returns The model, decoded once.
+     */
+    private loadModel;
+    /**
+     * Compares each word of a query with every word of the index.
+     *
+     * @param query - What the reader typed.
+     * @returns One entry per word of the query that carries meaning.
+     */
+    private queryWords;
+}

package/dist/embeddings/vectors.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Scales a vector to length 1.
+ *
+ * @param vector - Any vector. It is changed in place.
+ * @returns The same vector, or `null` when every value is zero.
+ */
+export declare function normalize(vector: Float32Array): Float32Array | null;
+/**
+ * Stores a vector of length 1 in one byte per value.
+ *
+ * @param unit - A vector of length 1.
+ * @returns The vector as signed bytes.
+ */
+export declare function toBytes(unit: Float32Array): Int8Array;
+/**
+ * Measures how alike two vectors are.
+ *
+ * @param query - A vector of length 1.
+ * @param rows - Stored vectors, one after the other, each `query.length` bytes.
+ * @param row - Index of the stored vector to compare with.
+ * @returns The cosine similarity: 1 for the same direction, 0 for unrelated.
+ */
+export declare function similarity(query: Float32Array, rows: Int8Array, row: number): number;
+/**
+ * Reads one stored vector back as numbers.
+ *
+ * @param rows - Stored vectors, one after the other.
+ * @param dimensions - Length of every vector.
+ * @returns The vector, scaled to length 1, or `null` when it is all zeros.
+ */
+export declare function rowOf(rows: Int8Array, row: number, dimensions: number): Float32Array | null;
+/**
+ * Stores small whole numbers in four bits each, two per byte. Halves the
+ * size of the model table.
+ *
+ * @param values - Numbers from -8 to 7.
+ * @returns One byte per two values. The last byte is padded when the count is odd.
+ * @throws {RangeError} When a value does not fit in four bits.
+ */
+export declare function packNibbles(values: Int8Array): Uint8Array;
+/**
+ * Reads back numbers that `packNibbles` stored.
+ *
+ * @param count - How many numbers to read.
+ * @returns The numbers, from -8 to 7.
+ */
+export declare function unpackNibbles(packed: Uint8Array, count: number): Int8Array;
+/**
+ * Encodes bytes as base64 text, for JSON.
+ *
+ * @returns Base64 text.
+ */
+export declare function toBase64(bytes: Int8Array | Uint8Array): string;
+/**
+ * Decodes base64 text into signed bytes.
+ *
+ * @param base64 - Text written by `toBase64`.
+ * @returns The bytes.
+ */
+export declare function fromBase64(base64: string): Int8Array;
+/**
+ * Packs a model table for a bundle.
+ *
+ * @param vectors - One value from -8 to 7 per dimension and word.
+ * @returns Base64 text of the values at four bits each.
+ */
+export declare function encodeTable(vectors: Int8Array): string;
+/**
+ * Reads a model table back out of a bundle.
+ *
+ * @param base64 - Text written by `encodeTable`.
+ * @param count - Number of values: words times dimensions.
+ * @returns One value per dimension and word.
+ */
+export declare function decodeTable(base64: string, count: number): Int8Array;

package/dist/embeddings/words.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Splits a text into lowercase words and single punctuation marks. Identifiers
+ * are split into their words, accents are dropped, and a short form from code
+ * is replaced with the word it stands for, so `srcDir` reads `source`, `directory`.
+ *
+ * @param text - Any text: prose, a name from code, a search query.
+ * @returns The words and marks, in text order.
+ */
+export declare function wordsOf(text: string): string[];
+/**
+ * Says whether a word from `wordsOf` is a punctuation mark.
+ *
+ * @param word - One item of the result of `wordsOf`.
+ * @returns `true` for a single punctuation character.
+ */
+export declare function isPunctuation(word: string): boolean;
+/**
+ * The words an embedding model has a vector for.
+ *
+ * Holds a map from each word to its row in the model table. `EmbeddingModel`
+ * creates one from its table and uses it to find the row of a word. The
+ * scanner side and the page look words up the same way, so a query meets the
+ * vectors it was meant for.
+ */
+export declare class WordVocabulary {
+    private readonly rows;
+    /**
+     * Creates a vocabulary.
+     *
+     * @param words - The words of the model. The index of a word is its row.
+     */
+    constructor(words: readonly string[]);
+    /**
+     * Finds the row that carries the meaning of a word: that of the word
+     * itself, else that of its stem when the word is a stem with a plain
+     * ending, as `sailings` is of `sailing`.
+     *
+     * @param word - A word from `wordsOf`.
+     * @returns The row, or `undefined` for a word the model does not have.
+     */
+    rowOf(word: string): number | undefined;
+}

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+export { type CreateDocsOptions, type CreateDocsResult, createDocs } from './api/create-docs';
+export { OptionsError, QueryError, SourceError } from './api/errors';
+export { type ListOptions, type ListResult, list } from './api/list';
+export type { WarningHandler } from './api/options';
+export { type RefsOptions, type RefsResult, refs } from './api/refs';
+export { type SearchOptions, type SearchResult, search } from './api/search';
+export { type ShowOptions, show } from './api/show';
+export type { EdgeKind, NodeKind, Section } from './model';
+export type { ContentReport, DatabaseObjectReport, DeclarationReport, NodeReport, NodeSummary, Reference, References, SearchHit, SearchMode, TaskReport, } from './retrieval';