@g3nomaly/analyze 0.0.0-stage → 0.1.1

package/dist/api/bundle-builder.d.ts

@@ -0,0 +1,94 @@
+import type { DocsBundle } from '../model';
+import { type FileLister, type Reporter, type RepositoryScanner, type Source, type TaskTreeSource } from '../scanner';
+import type { SemanticIndexer } from './embedding';
+import type { ScanOptions } from './scan';
+/**
+ * What a `BundleBuilder` works with.
+ *
+ * @property files - Lists the files of a directory.
+ * @property scanner - Reads the files of a scan into graph nodes.
+ * @property semantic - Embeds the nodes for search by meaning. `null` leaves that search out of the bundle.
+ * @property generator - Recorded in the bundle as the tool that produced it.
+ * @property reporter - Receives the problems that do not stop a scan, such as a config file that is not valid.
+ */
+export interface BundleBuilderServices {
+    readonly files: FileLister;
+    readonly scanner: RepositoryScanner;
+    readonly semantic: SemanticIndexer | null;
+    readonly generator: string;
+    readonly reporter: Reporter;
+}
+/**
+ * The directories of one scan, per section.
+ *
+ * @property skills - Directories the skills are read from.
+ * @property docs - Directories the docs are read from.
+ * @property code - Directories the code is read from.
+ * @property tasks - The task-manager tree of the repository. Empty when the repository has none.
+ */
+export interface ScanSources {
+    readonly skills: readonly Source[];
+    readonly docs: readonly Source[];
+    readonly code: readonly Source[];
+    readonly tasks: readonly TaskTreeSource[];
+}
+/**
+ * @brief Scans a repository into a bundle, with the file lister, scanner and embedder it holds.
+ *
+ * `createBuilder` in `src/api/services.ts` creates one per scan. `createDocs` writes its bundle into a site,
+ * and `ScanLoader` hands it to the query functions. Both first call `sources` to list the files of the scan,
+ * then `build` to read them; `ScanLoader` skips `build` when its cache has a bundle for the same files.
+ *
+ * @example
+ * const builder = new BundleBuilder({ files, scanner, semantic: null, generator: GENERATOR, reporter });
+ * const bundle = await builder.build(await builder.sources(scan), 'harbor');
+ */
+export declare class BundleBuilder {
+    private readonly services;
+    /**
+     * Creates a builder from the file lister, the scanner, the embedder and the reporter to use.
+     *
+     * @param services - What the builder works with.
+     */
+    constructor(services: BundleBuilderServices);
+    /**
+     * Lists the files of a scan without reading them.
+     *
+     * @param options - The repository and the directories to read skills and docs from.
+     * @returns The directories of each section with their files, and the task tree of the repository.
+     * @throws {SourceError} When the repository or one of the directories does not exist.
+     */
+    sources(options: ScanOptions): Promise<ScanSources>;
+    /**
+     * Reads the files of a scan and links what they declare.
+     *
+     * @param sources - The directories of each section, from `sources`.
+     * @param title - Title of the bundle.
+     * @returns The bundle, with a semantic index when the builder has an embedder.
+     */
+    build(sources: ScanSources, title: string): Promise<DocsBundle>;
+    /**
+     * @brief Finds the task-manager tree of a repository.
+     *
+     * The directory `skills.settings.filetreeRoot` of its `.g3n.config.json` names, `docs/task-manager` by default.
+     *
+     * @param repository - The repository to look in.
+     * @returns One tree with the files below its directory, or none when the directory does not exist.
+     */
+    private tasksIn;
+    /**
+     * Reads the directory of the task tree from the config of a repository, without writing it.
+     *
+     * @param repo - Absolute path of the repository.
+     * @returns The root-relative POSIX path of the tree. The default when the config is not valid.
+     */
+    private taskTreeDir;
+    /**
+     * Picks the directories one section is read from.
+     *
+     * @param repository - The repository source, read when no directories are given.
+     * @param directories - Absolute directories the caller passed. Empty means the repository.
+     * @returns The repository, or one source per directory.
+     */
+    private sourcesIn;
+}

package/dist/api/bundle-cache.d.ts

@@ -0,0 +1,73 @@
+import type { DocsBundle } from '../model';
+import type { ScanSources } from './bundle-builder';
+import type { ScanOptions } from './scan';
+/** Keeps the bundle of one scan between runs. */
+export interface BundleCache {
+    /**
+     * Returns the kept bundle when it was built from the same files.
+     *
+     * @param fingerprint - Fingerprint of the files as they are now.
+     * @returns The bundle, or `null` when nothing is kept or a file changed.
+     */
+    read(fingerprint: string): Promise<DocsBundle | null>;
+    /**
+     * Keeps a bundle, in place of the one kept before.
+     *
+     * @param fingerprint - Fingerprint of the files the bundle was built from.
+     * @param bundle - The bundle to keep.
+     */
+    write(fingerprint: string, bundle: DocsBundle): Promise<void>;
+}
+/**
+ * @brief Names the cache entry of a scan.
+ *
+ * Two scans share an entry when they read the same repository with the same options and the same version
+ * of the tool.
+ *
+ * @param scan - The options of the scan.
+ * @param generator - Name and version of the tool, such as `GENERATOR` in `src/api/services.ts`.
+ * @returns A file-name-safe key.
+ */
+export declare function cacheKey(scan: ScanOptions, generator: string): string;
+/**
+ * @brief Fingerprints the files of a scan by path, size and modification time, and
+ * the directory of its task tree. The fingerprint changes when a file is
+ * added, removed or saved, and when the config moves the task tree.
+ *
+ * @param sources - The directories of the scan with their files.
+ * @returns 64 hex digits.
+ */
+export declare function fingerprintOf(sources: ScanSources): Promise<string>;
+/**
+ * @brief Keeps a bundle in one file of a cache directory. The first line of the
+ * file is the fingerprint, the rest is the bundle as JSON.
+ *
+ * `openRetriever` creates one per query that scans a repository, in
+ * `analyze_out/cache` of that repository. The scan skips that directory, so
+ * a cache entry never changes the fingerprint of the next scan.
+ */
+export declare class FileBundleCache implements BundleCache {
+    private readonly directory;
+    private readonly file;
+    /**
+     * Creates the cache of one scan.
+     *
+     * @param directory - Cache directory. Created on the first write.
+     * @param key - Name of the entry, from `cacheKey`.
+     */
+    constructor(directory: string, key: string);
+    /**
+     * Reads the entry when its first line is the fingerprint.
+     *
+     * @param fingerprint - Fingerprint of the files as they are now.
+     * @returns The bundle, or `null` when the entry is missing, older or not readable.
+     */
+    read(fingerprint: string): Promise<DocsBundle | null>;
+    /**
+     * Writes the entry to a draft file and renames it into place.
+     *
+     * @param fingerprint - Fingerprint of the files the bundle was built from.
+     * @param bundle - The bundle to keep.
+     */
+    write(fingerprint: string, bundle: DocsBundle): Promise<void>;
+}

package/dist/api/bundle-loader.d.ts

@@ -0,0 +1,65 @@
+import { type DocsBundle } from '../model';
+import type { Reporter } from '../scanner';
+import type { BundleBuilder } from './bundle-builder';
+import { type BundleCache } from './bundle-cache';
+import type { ScanOptions } from './scan';
+/** Gets the bundle a query function reads. `loaderOf` in `src/api/source.ts` picks the implementation. */
+export interface BundleLoader {
+    /**
+     * Loads the bundle to query.
+     *
+     * @returns The bundle the queries read.
+     */
+    load(): Promise<DocsBundle>;
+}
+/**
+ * @brief Loads the bundle of a site that `createDocs` wrote, from the HTML file alone, without the repository the
+ * site was written from. `loaderOf` in `src/api/source.ts` creates one when a caller passes `from`
+ * (`--from` on the CLI); `openRetriever` loads it.
+ */
+export declare class SiteLoader implements BundleLoader {
+    private readonly file;
+    private readonly reporter;
+    /**
+     * Creates a loader for one site.
+     *
+     * @param file - Absolute path of the HTML file.
+     * @param reporter - Receives a warning when the site cannot be searched by meaning.
+     */
+    constructor(file: string, reporter: Reporter);
+    /**
+     * Reads the HTML file and takes the bundle out of it. Warns when its embeddings are of another format.
+     *
+     * @returns The bundle the site holds.
+     * @throws {SourceError} When the file is missing or is no analyze site.
+     */
+    load(): Promise<DocsBundle>;
+}
+/**
+ * @brief Loads the bundle of a repository: from the cache when no file changed
+ * since the last scan, else from a new scan, which it then keeps.
+ *
+ * `loaderOf` in `src/api/source.ts` creates one when a caller passes no `from`; `openRetriever` loads it.
+ */
+export declare class ScanLoader implements BundleLoader {
+    private readonly builder;
+    private readonly scan;
+    private readonly cache;
+    private readonly reporter;
+    /**
+     * Creates a loader for one repository.
+     *
+     * @param builder - Builds the bundle when the cache has none.
+     * @param scan - The repository to scan.
+     * @param cache - Keeps the bundle between runs. `null` scans every time.
+     * @param reporter - Receives a warning when the cache cannot be written.
+     */
+    constructor(builder: BundleBuilder, scan: ScanOptions, cache: BundleCache | null, reporter: Reporter);
+    /**
+     * Lists the files of the repository, then reads the cache or scans.
+     *
+     * @returns The bundle of the repository as it is now.
+     * @throws {SourceError} When the repository does not exist.
+     */
+    load(): Promise<DocsBundle>;
+}

package/dist/api/create-docs.d.ts

@@ -0,0 +1,91 @@
+import { z } from 'zod';
+import { type PackageRole } from '../model';
+/**
+ * Schema of the `createDocs` options.
+ *
+ * @property title - Site title. Default: the name of the repository directory.
+ */
+declare const createDocsSchema: z.ZodPipe<z.ZodObject<{
+    onWarning: z.ZodOptional<z.ZodCustom<import("./options").WarningHandler, import("./options").WarningHandler>>;
+    repo: z.ZodOptional<z.ZodString>;
+    skillDirs: z.ZodOptional<z.ZodReadonly<z.ZodArray<z.ZodString>>>;
+    docDirs: z.ZodOptional<z.ZodReadonly<z.ZodArray<z.ZodString>>>;
+    excludes: z.ZodOptional<z.ZodReadonly<z.ZodArray<z.ZodString>>>;
+    embeddings: z.ZodOptional<z.ZodBoolean>;
+    title: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>, z.ZodTransform<{
+    scan: import("./scan").ScanOptions;
+    title: string;
+    reporter: import("../scanner").Reporter;
+}, {
+    onWarning?: import("./options").WarningHandler | undefined;
+    repo?: string | undefined;
+    skillDirs?: readonly string[] | undefined;
+    docDirs?: readonly string[] | undefined;
+    excludes?: readonly string[] | undefined;
+    embeddings?: boolean | undefined;
+    title?: string | undefined;
+}>>;
+/** Options of `createDocs`: what to scan and the title of the site. */
+export type CreateDocsOptions = z.input<typeof createDocsSchema>;
+/**
+ * Where `createDocs` wrote the site and what the site holds.
+ *
+ * @property out - Absolute path of the site: `analyze_out/index.html` in the repository.
+ * @property bytes - Size of the site in bytes.
+ * @property skills - Skills of the bundle.
+ * @property docs - Pages of the docs section.
+ * @property files - Source files of the code section.
+ * @property configs - JSON, TOML and YAML files of the code section, OpenAPI documents and JSON Schemas included.
+ * @property packages - Packages of the code section, by role.
+ * @property symbols - Symbols of the source files.
+ * @property terms - Glossary terms.
+ * @property decisions - Decisions.
+ * @property operations - Operations of OpenAPI documents.
+ * @property definitions - Definitions of JSON Schemas and of OpenAPI documents.
+ * @property schemas - SQL files that create tables.
+ * @property tables - Tables of those files.
+ * @property catalogs - Directories of SQL files that create no table.
+ * @property queries - Queries and statements of those directories.
+ * @property plans - Plans of the task tree.
+ * @property tasks - Tasks of the task tree.
+ * @property completedTasks - Tasks with the status `completed`.
+ * @property links - Links between nodes: type references, mentions and the like.
+ * @property embedded - Nodes that search by meaning can find, and the model that embedded them. `null` without embeddings.
+ */
+export interface CreateDocsResult {
+    readonly out: string;
+    readonly bytes: number;
+    readonly skills: number;
+    readonly docs: number;
+    readonly files: number;
+    readonly configs: number;
+    readonly packages: Readonly<Record<PackageRole, number>>;
+    readonly symbols: number;
+    readonly terms: number;
+    readonly decisions: number;
+    readonly operations: number;
+    readonly definitions: number;
+    readonly schemas: number;
+    readonly tables: number;
+    readonly catalogs: number;
+    readonly queries: number;
+    readonly plans: number;
+    readonly tasks: number;
+    readonly completedTasks: number;
+    readonly links: number;
+    readonly embedded: {
+        readonly nodes: number;
+        readonly model: string;
+    } | null;
+}
+/**
+ * Scans a repository and writes its documentation site to `analyze_out/index.html` in the repository.
+ *
+ * @param options - What to scan and the title. Default: the working directory.
+ * @returns Where the site was written and what it holds.
+ * @throws {OptionsError} When the options are not valid.
+ * @throws {SourceError} When the repository or one of the directories does not exist.
+ */
+export declare function createDocs(options?: CreateDocsOptions): Promise<CreateDocsResult>;
+export {};

package/dist/api/embedding.d.ts

@@ -0,0 +1,41 @@
+import type { GraphNode, SemanticIndex } from '../model';
+/** Adds search by meaning to a bundle. `BundleBuilder.build` calls it when the scan has embeddings. */
+export interface SemanticIndexer {
+    /**
+     * Embeds the searchable nodes of a scan.
+     *
+     * @param nodes - Nodes of the scan to embed.
+     * @returns The index for `DocsBundle.semantic`.
+     */
+    index(nodes: readonly GraphNode[]): Promise<SemanticIndex>;
+}
+/**
+ * @brief Embeds nodes with the model that ships with the package.
+ *
+ * Holds the word table of the model, read on first use from `vectors.bin` and `vocab.txt` in `build/embedding/`
+ * when run from source, or beside the bundle when built. `shippedModel` in `src/api/services.ts` imports this
+ * module on first use and creates one for each scan with embeddings, with its own directory as `baseDir`.
+ */
+export declare class ShippedModelIndexer implements SemanticIndexer {
+    private readonly baseDir;
+    private table;
+    /**
+     * Creates an indexer that has not read the model yet.
+     *
+     * @param baseDir - Absolute directory that relative paths of the bundled model files resolve against.
+     */
+    constructor(baseDir: string);
+    /**
+     * Reads the model on the first call, then embeds the searchable nodes.
+     *
+     * @param nodes - Nodes of the scan to embed.
+     * @returns The index for `DocsBundle.semantic`.
+     */
+    index(nodes: readonly GraphNode[]): Promise<SemanticIndex>;
+    /**
+     * Reads the two files of the model and unpacks the four-bit vectors.
+     *
+     * @returns The word table of the model, with its vectors unpacked.
+     */
+    private load;
+}

package/dist/api/errors.d.ts

@@ -0,0 +1,35 @@
+/**
+ * @brief Options a caller passed that do not form a valid call.
+ *
+ * Holds one issue per problem in `issues`, and its message lists them all. Thrown by `parseOptions` in
+ * `src/api/options.ts`; `src/cli/index.ts` prints the usage and exits with 2 for it. Created with
+ * `new OptionsError(["limit: must be positive"])`.
+ */
+export declare class OptionsError extends Error {
+    readonly issues: readonly string[];
+    readonly name = "OptionsError";
+    /**
+     * Creates the error from what is wrong.
+     *
+     * @param issues - One line per problem: the option and what is wrong with it.
+     */
+    constructor(issues: readonly string[]);
+}
+/**
+ * @brief Error for a query that cannot be answered, such as one for a node that does not exist.
+ *
+ * Thrown by `src/api/target.ts` and `src/api/search.ts`; the CLI exits with 1 for it. Created with the
+ * message, e.g. `new QueryError('No node is called "x".')`.
+ */
+export declare class QueryError extends Error {
+    readonly name = "QueryError";
+}
+/**
+ * @brief Error for a repository, a directory or a site file that is missing or cannot be read.
+ *
+ * Thrown by the bundle builder, loader and site reader in `src/api/`; the CLI exits with 1 for it. Created
+ * with the message, e.g. `new SourceError('Site not found: /path/site.html')`.
+ */
+export declare class SourceError extends Error {
+    readonly name = "SourceError";
+}

package/dist/api/list.d.ts

@@ -0,0 +1,106 @@
+import { z } from 'zod';
+import type { NodeFilter, NodeSummary, Retriever } from '../retrieval';
+/**
+ * Schema of the `list` options.
+ *
+ * @property limit - Most rows. Absent lists everything.
+ */
+declare const listSchema: z.ZodPipe<z.ZodObject<{
+    onWarning: z.ZodOptional<z.ZodCustom<import("./options").WarningHandler, import("./options").WarningHandler>>;
+    kinds: z.ZodOptional<z.ZodReadonly<z.ZodArray<z.ZodEnum<{
+        class: "class";
+        decision: "decision";
+        definition: "definition";
+        enum: "enum";
+        field: "field";
+        file: "file";
+        folder: "folder";
+        function: "function";
+        interface: "interface";
+        namespace: "namespace";
+        operation: "operation";
+        package: "package";
+        parameter: "parameter";
+        phase: "phase";
+        plan: "plan";
+        query: "query";
+        struct: "struct";
+        table: "table";
+        task: "task";
+        term: "term";
+        trait: "trait";
+        type: "type";
+    }>>>>;
+    section: z.ZodOptional<z.ZodEnum<{
+        code: "code";
+        database: "database";
+        docs: "docs";
+        skills: "skills";
+        tasks: "tasks";
+    }>>;
+    repo: z.ZodOptional<z.ZodString>;
+    skillDirs: z.ZodOptional<z.ZodReadonly<z.ZodArray<z.ZodString>>>;
+    docDirs: z.ZodOptional<z.ZodReadonly<z.ZodArray<z.ZodString>>>;
+    excludes: z.ZodOptional<z.ZodReadonly<z.ZodArray<z.ZodString>>>;
+    embeddings: z.ZodOptional<z.ZodBoolean>;
+    from: z.ZodOptional<z.ZodString>;
+    cache: z.ZodOptional<z.ZodBoolean>;
+    limit: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>, z.ZodTransform<{
+    source: import("./source").BundleSource;
+    reporter: import("../scanner").Reporter;
+    request: {
+        filter: NodeFilter;
+        limit: number | null;
+    };
+}, {
+    onWarning?: import("./options").WarningHandler | undefined;
+    kinds?: readonly ("class" | "decision" | "definition" | "enum" | "field" | "file" | "folder" | "function" | "interface" | "namespace" | "operation" | "package" | "parameter" | "phase" | "plan" | "query" | "struct" | "table" | "task" | "term" | "trait" | "type")[] | undefined;
+    section?: "code" | "database" | "docs" | "skills" | "tasks" | undefined;
+    repo?: string | undefined;
+    skillDirs?: readonly string[] | undefined;
+    docDirs?: readonly string[] | undefined;
+    excludes?: readonly string[] | undefined;
+    embeddings?: boolean | undefined;
+    from?: string | undefined;
+    cache?: boolean | undefined;
+    limit?: number | undefined;
+}>>;
+/** Options of `list`: the filter, the limit, and the bundle to list. */
+export type ListOptions = z.input<typeof listSchema>;
+/**
+ * One `list` on a loaded bundle.
+ *
+ * @property filter - Narrows the nodes that count.
+ * @property limit - Most rows. `null` lists everything.
+ */
+export interface ListRequest {
+    readonly filter: NodeFilter;
+    readonly limit: number | null;
+}
+/**
+ * What `list` finds. The CLI prints it with `list --json`.
+ *
+ * @property nodes - The nodes, in the order of the navigation.
+ */
+export interface ListResult {
+    readonly nodes: readonly NodeSummary[];
+}
+/**
+ * Lists the pages and entries of a loaded bundle.
+ *
+ * @param retriever - The loaded bundle to list.
+ * @param request - The filter and the limit.
+ * @returns The nodes, in the order of the navigation.
+ */
+export declare function listIn(retriever: Retriever, request: ListRequest): ListResult;
+/**
+ * Lists pages and entries.
+ *
+ * @param options - The filter, the limit, and the repository or site to list. Default: every node of the working directory.
+ * @returns The nodes, in the order of the navigation.
+ * @throws {OptionsError} When the options are not valid.
+ * @throws {SourceError} When the repository or the site cannot be read.
+ */
+export declare function list(options?: ListOptions): Promise<ListResult>;
+export {};

package/dist/api/options.d.ts

@@ -0,0 +1,74 @@
+import { z } from 'zod';
+import type { NodeFilter } from '../retrieval';
+import { type Reporter } from '../scanner';
+/** Receives problems that do not stop a call, such as a scan that could not be cached. */
+export type WarningHandler = (message: string) => void;
+/** The option every function takes to hear about warnings. Without it, warnings are dropped. */
+export declare const warningShape: {
+    onWarning: z.ZodOptional<z.ZodCustom<WarningHandler, WarningHandler>>;
+};
+/**
+ * The options that narrow the nodes a query looks at.
+ *
+ * @property kinds - Only nodes of these kinds. Empty or absent keeps every kind.
+ * @property section - Only nodes of this section.
+ */
+export declare const filterShape: {
+    kinds: z.ZodOptional<z.ZodReadonly<z.ZodArray<z.ZodEnum<{
+        class: "class";
+        decision: "decision";
+        definition: "definition";
+        enum: "enum";
+        field: "field";
+        file: "file";
+        folder: "folder";
+        function: "function";
+        interface: "interface";
+        namespace: "namespace";
+        operation: "operation";
+        package: "package";
+        parameter: "parameter";
+        phase: "phase";
+        plan: "plan";
+        query: "query";
+        struct: "struct";
+        table: "table";
+        task: "task";
+        term: "term";
+        trait: "trait";
+        type: "type";
+    }>>>>;
+    section: z.ZodOptional<z.ZodEnum<{
+        code: "code";
+        database: "database";
+        docs: "docs";
+        skills: "skills";
+        tasks: "tasks";
+    }>>;
+};
+/** The filter options after validation: `kinds` and `section`, both optional. */
+type FilterFields = z.output<z.ZodObject<typeof filterShape>>;
+/**
+ * Turns validated filter options into the filter the retriever takes.
+ *
+ * @param fields - Validated filter options, from `filterShape`.
+ * @returns The filter the retriever takes. Absent options keep every node.
+ */
+export declare function filterOf(fields: FilterFields): NodeFilter;
+/**
+ * Turns the warning option into the reporter the services take.
+ *
+ * @param onWarning - The caller's handler, or `undefined` to drop the warnings.
+ * @returns A reporter that hands each warning to the handler, or drops it.
+ */
+export declare function reporterOf(onWarning: WarningHandler | undefined): Reporter;
+/**
+ * Validates the options a caller passed against the schema of a function.
+ *
+ * @param schema - The schema of the options, with its transform.
+ * @param input - The options the caller passed, not yet validated.
+ * @returns The options after the transform.
+ * @throws {OptionsError} When the options do not fit the schema. Its `issues` name each problem.
+ */
+export declare function parseOptions<Schema extends z.ZodType>(schema: Schema, input: unknown): z.output<Schema>;
+export {};