@tryformation/querylight-cli 0.1.1

package/dist/ingest/adapters/crawl4ai-adapter.d.ts

@@ -0,0 +1 @@
+export declare function crawlWithCrawl4Ai(): Promise<never>;

package/dist/ingest/adapters/directory-adapter.d.ts

@@ -0,0 +1,2 @@
+import type { Source } from "../../types/models.js";
+export declare function listDirectoryFiles(source: Source): Promise<string[]>;

package/dist/ingest/adapters/file-adapter.d.ts

@@ -0,0 +1,16 @@
+import type { DocumentRecord, Source } from "../../types/models.js";
+export declare function ingestFile({ workspacePath, source, filePath, previous }: {
+    workspacePath: string;
+    source: Source;
+    filePath: string;
+    previous?: DocumentRecord;
+}): Promise<DocumentRecord>;
+export declare function ingestInlineContent({ workspacePath, source, content, title, uri, previous }: {
+    workspacePath: string;
+    source: Source;
+    content: string;
+    title: string;
+    uri: string;
+    previous?: DocumentRecord;
+}): Promise<DocumentRecord>;
+export declare function reprocessStoredDocument(document: DocumentRecord, source: Source): Promise<DocumentRecord | null>;

package/dist/ingest/adapters/rss-adapter.d.ts

@@ -0,0 +1,7 @@
+import type { Source } from "../../types/models.js";
+export type ParsedFeedItem = {
+    url: string;
+    title: string;
+    publicationDate: string | null;
+};
+export declare function parseRssFeedDocument(xml: string, source: Source): Promise<ParsedFeedItem[]>;

package/dist/ingest/adapters/url-adapter.d.ts

@@ -0,0 +1,11 @@
+import type { DocumentRecord, Source } from "../../types/models.js";
+export type FetchRemoteDocumentOptions = {
+    workspacePath: string;
+    source: Source;
+    url: string;
+    previous?: DocumentRecord;
+    sourceUri?: string;
+    publicationDate?: string | null;
+};
+export declare function fetchUrlDocument({ workspacePath, source, url, previous, sourceUri, publicationDate }: FetchRemoteDocumentOptions): Promise<DocumentRecord>;
+export declare function reprocessRemoteDocument(document: DocumentRecord, source: Source): Promise<DocumentRecord | null>;

package/dist/ingest/adapters/website-adapter.d.ts

@@ -0,0 +1,2 @@
+import type { Source } from "../../types/models.js";
+export declare function crawlWebsite(source: Source): Promise<string[]>;

package/dist/ingest/document-utils.d.ts

@@ -0,0 +1,24 @@
+import type { DocumentRecord, Metadata, PrimitiveMetadata, Source } from "../types/models.js";
+export declare function buildDocumentMetadata({ source, sourceUri, publicationDate, crawledAt, indexedAt, extra }: {
+    source: Source;
+    sourceUri: string;
+    publicationDate?: string | null;
+    crawledAt?: string;
+    indexedAt?: string;
+    extra?: Record<string, PrimitiveMetadata | undefined>;
+}): Metadata;
+export declare function writeNormalizedDocument({ documentId, sourceId, title, uri, sourceUri, publicationDate, crawledAt, indexedAt, contentHash, lastChangedAt, normalizedPath, markdown }: {
+    documentId: string;
+    sourceId: string;
+    title: string;
+    uri: string;
+    sourceUri: string;
+    publicationDate?: string | null;
+    crawledAt?: string;
+    indexedAt?: string;
+    contentHash: string;
+    lastChangedAt: string;
+    normalizedPath: string;
+    markdown: string;
+}): Promise<void>;
+export declare function deleteDocumentArtifacts(document: DocumentRecord): Promise<void>;

package/dist/ingest/extractors/docx-extractor.d.ts

@@ -0,0 +1 @@
+export declare function extractDocx(filePath: string): Promise<string>;

package/dist/ingest/extractors/html-extractor.d.ts

@@ -0,0 +1,5 @@
+export declare function extractHtmlToMarkdown(html: string): {
+    markdown: string;
+    title: string;
+};
+export declare function extractPublicationDateFromHtml(html: string): string | null;

package/dist/ingest/extractors/markdown-extractor.d.ts

@@ -0,0 +1 @@
+export declare function extractMarkdown(filePath: string): Promise<string>;

package/dist/ingest/extractors/pdf-extractor.d.ts

@@ -0,0 +1 @@
+export declare function extractPdf(filePath: string): Promise<string>;

package/dist/ingest/extractors/text-extractor.d.ts

@@ -0,0 +1 @@
+export declare function extractText(filePath: string): Promise<string>;

package/dist/ingest/ingest-service.d.ts

@@ -0,0 +1,23 @@
+export declare function ingestSources({ workspacePath, sourceIds, changedOnly }: {
+    workspacePath: string;
+    sourceIds?: string[];
+    changedOnly?: boolean;
+}): Promise<{
+    runId: string;
+    documents: {
+        added: number;
+        changed: number;
+        unchanged: number;
+        failed: number;
+    };
+    processedSources: number;
+}>;
+export declare function reprocessDocuments({ workspacePath, sourceId, documentId }: {
+    workspacePath: string;
+    sourceId?: string;
+    documentId?: string;
+}): Promise<{
+    runId: string;
+    documentsReprocessed: number;
+    documentsSkipped: number;
+}>;

package/dist/normalize/boilerplate.d.ts

@@ -0,0 +1 @@
+export declare function stripBoilerplate(html: string): string;

package/dist/normalize/normalize-markdown.d.ts

@@ -0,0 +1,2 @@
+export declare function normalizeWhitespace(text: string): string;
+export declare function withFrontmatter(metadata: Record<string, unknown>, body: string): string;

package/dist/query/context-builder.d.ts

@@ -0,0 +1,8 @@
+import type { ContextResponseData } from "../types/models.js";
+export declare function createContext({ workspacePath, query, topK, maxChars, retrievalMode }: {
+    workspacePath: string;
+    query: string;
+    topK: number;
+    maxChars: number;
+    retrievalMode?: import("../types/models.js").RetrievalMode;
+}): Promise<ContextResponseData>;

package/dist/query/related-service.d.ts

@@ -0,0 +1,6 @@
+import type { RelatedDocumentsResponseData } from "../types/models.js";
+export declare function findRelatedDocuments({ workspacePath, document, topK }: {
+    workspacePath: string;
+    document: string;
+    topK: number;
+}): Promise<RelatedDocumentsResponseData>;

package/dist/query/search-service.d.ts

@@ -0,0 +1,31 @@
+import type { RetrievalMode, SearchResponseData } from "../types/models.js";
+type SearchDateField = "publicationDate" | "firstSeenAt" | "lastSeenAt" | "lastChangedAt" | "crawledAt";
+type SearchDateRange = {
+    field: SearchDateField;
+    from?: string;
+    to?: string;
+};
+export declare function searchIndex({ workspacePath, query, topK, sourceId, sourceIds, sourceName, sourceNames, sourceType, sourceTypes, uriPrefix, uriPrefixes, hasPublicationDate, tag, tags, metadata, dateRanges, retrievalMode, showChunks }: {
+    workspacePath: string;
+    query: string;
+    topK: number;
+    sourceId?: string;
+    sourceIds?: string[];
+    sourceName?: string;
+    sourceNames?: string[];
+    sourceType?: string;
+    sourceTypes?: string[];
+    uriPrefix?: string;
+    uriPrefixes?: string[];
+    hasPublicationDate?: boolean;
+    tag?: string;
+    tags?: string[];
+    metadata?: Array<{
+        key: string;
+        value: string;
+    }>;
+    dateRanges?: SearchDateRange[];
+    retrievalMode?: RetrievalMode;
+    showChunks?: boolean;
+}): Promise<SearchResponseData>;
+export {};

package/dist/report/diff-service.d.ts

@@ -0,0 +1,23 @@
+export declare function diffWorkspace({ workspacePath, sourceId, documentId, since }: {
+    workspacePath: string;
+    sourceId?: string;
+    documentId?: string;
+    since?: string;
+}): Promise<{
+    changedDocuments: Array<{
+        id: string;
+        title: string;
+        uri: string;
+        sourceId: string;
+        previousHash?: string;
+        currentHash: string;
+    }>;
+}>;
+export declare function renderChangeReport(diff: {
+    changedDocuments: Array<{
+        id: string;
+        title: string;
+        uri: string;
+        sourceId: string;
+    }>;
+}): string;

package/dist/sources/source-model.d.ts

@@ -0,0 +1 @@
+export type { Source, SourceType, CrawlConfig } from "../types/models.js";

package/dist/sources/source-store.d.ts

@@ -0,0 +1,7 @@
+import type { Source } from "../types/models.js";
+export declare function listSources(workspacePath: string): Promise<Source[]>;
+export declare function addSource(workspacePath: string, source: Omit<Source, "id"> & {
+    id?: string;
+}): Promise<Source>;
+export declare function updateSource(workspacePath: string, sourceId: string, patch: Partial<Source>): Promise<Source>;
+export declare function removeSource(workspacePath: string, sourceId: string): Promise<void>;

package/dist/types/models.d.ts

@@ -0,0 +1,309 @@
+export type SourceType = "url" | "website" | "rss" | "file" | "directory" | "markdown" | "text";
+export type PrimitiveMetadata = string | number | boolean | string[] | null;
+export type Metadata = Record<string, PrimitiveMetadata>;
+export type RetrievalMode = "lexical" | "dense" | "sparse" | "hybrid";
+export type CrawlConfig = {
+    maxDepth?: number;
+    maxPages?: number;
+    includePatterns?: string[];
+    excludePatterns?: string[];
+    obeyRobotsTxt?: boolean;
+    userAgent?: string;
+    rateLimitMs?: number;
+    useSitemap?: boolean;
+    renderJs?: boolean;
+    retentionDays?: number;
+    fetchArticles?: boolean;
+};
+export type HttpCacheMetadata = {
+    etag?: string;
+    lastModified?: string;
+    cacheControl?: string;
+    expires?: string | null;
+    lastValidatedAt?: string;
+    lastStatus?: number;
+};
+export type Source = {
+    id: string;
+    type: SourceType;
+    name: string;
+    uri: string;
+    enabled: boolean;
+    tags: string[];
+    metadata: Metadata;
+    crawl?: CrawlConfig;
+    createdAt: string;
+    updatedAt: string;
+};
+export type DocumentRecord = {
+    id: string;
+    sourceId: string;
+    sourceType: SourceType;
+    title: string;
+    uri: string;
+    sourceUri: string;
+    canonicalUri?: string;
+    mimeType: string;
+    language?: string;
+    rawPath?: string;
+    normalizedPath: string;
+    contentHash: string;
+    metadata: Metadata;
+    publicationDate?: string | null;
+    crawledAt?: string;
+    firstSeenAt: string;
+    lastSeenAt: string;
+    lastChangedAt: string;
+    indexedAt?: string;
+    httpCache?: HttpCacheMetadata;
+};
+export type ChunkRecord = {
+    id: string;
+    documentId: string;
+    sourceId: string;
+    title: string;
+    uri: string;
+    headingPath: string[];
+    text: string;
+    tokenEstimate?: number;
+    contentHash: string;
+    metadata: Metadata;
+    firstSeenAt: string;
+    lastSeenAt: string;
+    lastChangedAt: string;
+};
+export type IndexMetadata = {
+    id: string;
+    createdAt: string;
+    querylightVersion: string;
+    kbVersion: string;
+    documentCount: number;
+    chunkCount: number;
+    sourceCount: number;
+    fields: string[];
+    embeddingModel?: string;
+    sparseVectorModel?: string;
+    indexHash: string;
+};
+export type DenseVectorModelConfig = {
+    enabled: boolean;
+    modelId: string;
+    cacheDir: string;
+    indexHashTables: number;
+    indexRandomSeed: number;
+    chunkTextMode: "title-heading-text";
+};
+export type SparseVectorModelConfig = {
+    enabled: boolean;
+    modelId: string;
+    cacheDir: string;
+    documentTopTokens: number;
+    queryEncoding: "tokenizer-token-weights";
+    documentEncoding: "masked-lm-max-log1p-relu";
+    chunkTextMode: "title-heading-text";
+};
+export type DenseVectorMetadata = {
+    createdAt: string;
+    modelId: string;
+    dimensions: number;
+    hashTables: number;
+    randomSeed: number;
+    chunkCount: number;
+    indexHash: string;
+};
+export type SparseVectorMetadata = {
+    createdAt: string;
+    modelId: string;
+    vocabularySize: number;
+    documentTopTokens: number;
+    queryEncoding: "tokenizer-token-weights";
+    documentEncoding: "masked-lm-max-log1p-relu";
+    chunkCount: number;
+    indexHash: string;
+};
+export type DenseVectorRecord = {
+    chunkId: string;
+    documentId: string;
+    sourceId: string;
+    title: string;
+    uri: string;
+    headingPath: string[];
+    text: string;
+    embedding: number[];
+};
+export type SparseVectorRecord = {
+    chunkId: string;
+    documentId: string;
+    sourceId: string;
+    title: string;
+    uri: string;
+    headingPath: string[];
+    text: string;
+    vector: Record<string, number>;
+};
+export type DenseVectorPayload = {
+    metadata: DenseVectorMetadata;
+    indexState: object;
+    chunks: DenseVectorRecord[];
+};
+export type SparseVectorPayload = {
+    metadata: SparseVectorMetadata;
+    indexState: object;
+    chunks: SparseVectorRecord[];
+    queryTokenWeights: number[];
+};
+export type WorkspaceConfig = {
+    workspaceVersion: number;
+    index: {
+        name: string;
+        fields: Record<string, {
+            type: string;
+            weight?: number;
+        }>;
+        chunking: {
+            maxChars: number;
+            overlapChars: number;
+            minChars: number;
+            splitOnHeadings: boolean;
+        };
+    };
+    rag: {
+        defaultTopK: number;
+        maxContextChars: number;
+        citationStyle: "markdown";
+    };
+    retrieval: {
+        defaultMode: RetrievalMode;
+        dense: DenseVectorModelConfig;
+        sparse: SparseVectorModelConfig;
+    };
+    crawler: {
+        defaultUserAgent: string;
+        obeyRobotsTxt: boolean;
+        rateLimitMs: number;
+        renderJs: boolean;
+        retentionDays: number;
+        fetchArticles: boolean;
+    };
+    limits: {
+        maxFileSizeMb: number;
+        maxPagesPerSource: number;
+        maxTotalChunks: number;
+    };
+};
+export type CommandError = {
+    code: string;
+    message: string;
+    details?: unknown;
+};
+export type CommandResponse<T> = {
+    ok: boolean;
+    command: string;
+    workspace: string;
+    version: string;
+    data?: T;
+    error?: CommandError;
+};
+export type SearchResult = {
+    chunkId: string;
+    documentId: string;
+    sourceId: string;
+    sourceType: SourceType;
+    score: number;
+    title: string;
+    uri: string;
+    headingPath: string[];
+    snippet: string;
+    text?: string;
+    publicationDate?: string | null;
+    firstSeenAt: string;
+    lastSeenAt: string;
+    lastChangedAt: string;
+    metadata: Record<string, unknown>;
+};
+export type SearchResponseData = {
+    retrievalMode?: RetrievalMode;
+    results: SearchResult[];
+};
+export type RelatedDocumentResult = {
+    documentId: string;
+    sourceId: string;
+    score: number;
+    title: string;
+    uri: string;
+    metadata: Record<string, unknown>;
+};
+export type RelatedDocumentsResponseData = {
+    sourceDocument: {
+        documentId: string;
+        sourceId: string;
+        title: string;
+        uri: string;
+    };
+    retrievalMode: "dense";
+    results: RelatedDocumentResult[];
+};
+export type ContextSource = {
+    chunkId: string;
+    documentId: string;
+    sourceId: string;
+    title: string;
+    uri: string;
+    headingPath: string[];
+    text: string;
+    metadata: Record<string, unknown>;
+};
+export type ContextResponseData = {
+    retrievalMode?: RetrievalMode;
+    markdown: string;
+    sources: ContextSource[];
+};
+export type ModelPullResponse = {
+    dense?: {
+        pulled: boolean;
+        modelId: string;
+        cacheDir: string;
+    };
+    sparse?: {
+        pulled: boolean;
+        modelId: string;
+        cacheDir: string;
+    };
+};
+export type ModelStatusResponse = {
+    dense: {
+        configured: boolean;
+        modelId: string;
+        cacheDir: string;
+        available: boolean;
+        artifactExists: boolean;
+    };
+    sparse: {
+        configured: boolean;
+        modelId: string;
+        cacheDir: string;
+        uvAvailable: boolean;
+        available: boolean;
+        artifactExists: boolean;
+    };
+};
+export type RunRecord = {
+    id: string;
+    kind: string;
+    createdAt: string;
+    success: boolean;
+    summary: Record<string, unknown>;
+    failures?: Array<{
+        sourceId: string;
+        uri: string;
+        message: string;
+    }>;
+    documentsSnapshot?: Array<{
+        id: string;
+        title: string;
+        uri: string;
+        contentHash: string;
+        lastChangedAt: string;
+        sourceId: string;
+    }>;
+};

package/dist/vector/dense.d.ts

@@ -0,0 +1,13 @@
+import type { DenseVectorPayload, WorkspaceConfig } from "../types/models.js";
+export declare function setDenseEmbedderFactoryForTests(factory: ((cacheDir: string, modelId: string) => Promise<(text: string) => Promise<number[]>>) | null): void;
+export declare function pullDenseModel(workspacePath: string, config: WorkspaceConfig["retrieval"]["dense"]): Promise<void>;
+export declare function buildDenseVectors({ workspacePath, config }: {
+    workspacePath: string;
+    config: WorkspaceConfig["retrieval"]["dense"];
+}): Promise<DenseVectorPayload>;
+export declare function denseQuery({ workspacePath, config, query, topK }: {
+    workspacePath: string;
+    config: WorkspaceConfig["retrieval"]["dense"];
+    query: string;
+    topK: number;
+}): Promise<Array<[string, number]>>;

package/dist/vector/runtime.d.ts

@@ -0,0 +1,18 @@
+import type { SparseVectorModelConfig } from "../types/models.js";
+export declare function resolveCacheDir(workspacePath: string, configuredPath: string): string;
+export declare function packageRootFromImportMeta(importMetaUrl: string): string;
+export declare function sparseScriptPath(importMetaUrl: string): Promise<string>;
+export declare function ensureUvAvailable(): Promise<void>;
+export declare function runSparsePython({ workspacePath, config, payload, importMetaUrl }: {
+    workspacePath: string;
+    config: SparseVectorModelConfig;
+    payload: object;
+    importMetaUrl: string;
+}): Promise<string>;
+export declare function getDenseTransformersRuntime(cacheDir: string): Promise<{
+    env: {
+        cacheDir: string;
+        allowLocalModels: boolean;
+    };
+    pipeline: typeof import("@huggingface/transformers").pipeline;
+}>;

package/dist/vector/service.d.ts

@@ -0,0 +1,26 @@
+import type { DenseVectorPayload, SparseVectorPayload, WorkspaceConfig } from "../types/models.js";
+export declare function resolveModelPullPlan({ pullDenseFlag, pullSparseFlag, uvAvailable }: {
+    pullDenseFlag: boolean;
+    pullSparseFlag: boolean;
+    uvAvailable: boolean;
+}): {
+    pullDense: boolean;
+    pullSparse: boolean;
+};
+export declare function buildVectorArtifacts({ workspacePath, config, denseOverride, sparseOverride, buildAvailableModels }: {
+    workspacePath: string;
+    config: WorkspaceConfig;
+    denseOverride?: boolean;
+    sparseOverride?: boolean;
+    buildAvailableModels?: boolean;
+}): Promise<{
+    dense?: DenseVectorPayload;
+    sparse?: SparseVectorPayload;
+}>;
+export declare function pullModels({ workspacePath, config, pullDense, pullSparse }: {
+    workspacePath: string;
+    config: WorkspaceConfig;
+    pullDense: boolean;
+    pullSparse: boolean;
+}): Promise<void>;
+export declare function getModelStatus(workspacePath: string, config: WorkspaceConfig): Promise<import("../index.js").ModelStatusResponse>;

package/dist/vector/sparse.d.ts

@@ -0,0 +1,19 @@
+import { type SparseVector } from "@tryformation/querylight-ts";
+import type { ChunkRecord, SparseVectorPayload, SparseVectorRecord, WorkspaceConfig } from "../types/models.js";
+export declare function setSparseQueryEncoderFactoryForTests(factory: ((cacheDir: string, modelId: string, queryTokenWeights: number[]) => Promise<(text: string) => Promise<SparseVector>>) | null): void;
+export declare function setSparseDocumentBuilderFactoryForTests(factory: ((workspacePath: string, config: WorkspaceConfig["retrieval"]["sparse"], chunks: ChunkRecord[]) => Promise<{
+    queryTokenWeights: number[];
+    vocabularySize: number;
+    chunks: SparseVectorRecord[];
+}>) | null): void;
+export declare function pullSparseModel(workspacePath: string, config: WorkspaceConfig["retrieval"]["sparse"]): Promise<void>;
+export declare function buildSparseVectors({ workspacePath, config }: {
+    workspacePath: string;
+    config: WorkspaceConfig["retrieval"]["sparse"];
+}): Promise<SparseVectorPayload>;
+export declare function sparseQuery({ workspacePath, config, query, topK }: {
+    workspacePath: string;
+    config: WorkspaceConfig["retrieval"]["sparse"];
+    query: string;
+    topK: number;
+}): Promise<Array<[string, number]>>;

package/dist/vector/store.d.ts

@@ -0,0 +1,20 @@
+import type { DenseVectorPayload, ModelStatusResponse, SparseVectorPayload } from "../types/models.js";
+export declare function denseVectorPath(workspacePath: string): string;
+export declare function denseMetaPath(workspacePath: string): string;
+export declare function sparseVectorPath(workspacePath: string): string;
+export declare function sparseMetaPath(workspacePath: string): string;
+export declare function writeDensePayload(workspacePath: string, payload: DenseVectorPayload): Promise<void>;
+export declare function readDensePayload(workspacePath: string): Promise<DenseVectorPayload>;
+export declare function writeSparsePayload(workspacePath: string, payload: SparseVectorPayload): Promise<void>;
+export declare function readSparsePayload(workspacePath: string): Promise<SparseVectorPayload>;
+export declare function writeDensePullMarker(workspacePath: string, value: object): Promise<void>;
+export declare function writeSparsePullMarker(workspacePath: string, value: object): Promise<void>;
+export declare function buildModelStatus(workspacePath: string, dense: {
+    enabled: boolean;
+    modelId: string;
+    cacheDir: string;
+}, sparse: {
+    enabled: boolean;
+    modelId: string;
+    cacheDir: string;
+}, uvAvailable: boolean): Promise<ModelStatusResponse>;

package/dist/vector/text.d.ts

@@ -0,0 +1,3 @@
+import type { ChunkRecord } from "../types/models.js";
+export declare function createDenseChunkText(chunk: ChunkRecord): string;
+export declare function createSparseChunkText(chunk: ChunkRecord): string;