@tryformation/querylight-cli 0.1.1 → 0.2.0

package/dist/ingest/adapters/website-adapter.d.ts

@@ -1,2 +1,7 @@
+import { type ProgressHandler } from "../../core/progress.js";
 import type { Source } from "../../types/models.js";
-export declare function crawlWebsite(source: Source): Promise<string[]>;
+export declare function crawlWebsite(source: Source, defaults: {
+    userAgent: string;
+    rateLimitMs: number;
+    maxConcurrentRequests: number;
+}, progress?: ProgressHandler): Promise<string[]>;

package/dist/ingest/adapters/website-feed-discovery.d.ts

@@ -0,0 +1,6 @@
+export type WebsiteFeedDiscovery = {
+    feedUrl: string;
+    discoveredBy: "declared" | "common";
+    excludePrefix?: string;
+};
+export declare function discoverWebsiteFeed(websiteUrl: string, userAgent: string): Promise<WebsiteFeedDiscovery | null>;

package/dist/ingest/extractors/html-extractor.d.ts

@@ -2,4 +2,5 @@ export declare function extractHtmlToMarkdown(html: string): {
     markdown: string;
     title: string;
 };
+export declare function extractCanonicalUriFromHtml(html: string, baseUrl: string): string | null;
 export declare function extractPublicationDateFromHtml(html: string): string | null;

package/dist/ingest/ingest-service.d.ts

@@ -1,7 +1,9 @@
-export declare function ingestSources({ workspacePath, sourceIds, changedOnly }: {
+import { type ProgressHandler } from "../core/progress.js";
+export declare function ingestSources({ workspacePath, sourceIds, changedOnly, progress }: {
     workspacePath: string;
     sourceIds?: string[];
     changedOnly?: boolean;
+    progress?: ProgressHandler;
 }): Promise<{
     runId: string;
     documents: {
@@ -12,10 +14,11 @@ export declare function ingestSources({ workspacePath, sourceIds, changedOnly }:
     };
     processedSources: number;
 }>;
-export declare function reprocessDocuments({ workspacePath, sourceId, documentId }: {
+export declare function reprocessDocuments({ workspacePath, sourceId, documentId, progress }: {
     workspacePath: string;
     sourceId?: string;
     documentId?: string;
+    progress?: ProgressHandler;
 }): Promise<{
     runId: string;
     documentsReprocessed: number;

package/dist/types/models.d.ts

@@ -5,6 +5,7 @@ export type RetrievalMode = "lexical" | "dense" | "sparse" | "hybrid";
 export type CrawlConfig = {
     maxDepth?: number;
     maxPages?: number;
+    maxConcurrentRequests?: number;
     includePatterns?: string[];
     excludePatterns?: string[];
     obeyRobotsTxt?: boolean;
@@ -181,6 +182,7 @@ export type WorkspaceConfig = {
         defaultUserAgent: string;
         obeyRobotsTxt: boolean;
         rateLimitMs: number;
+        maxConcurrentRequests: number;
         renderJs: boolean;
         retentionDays: number;
         fetchArticles: boolean;
@@ -212,7 +214,6 @@ export type SearchResult = {
     score: number;
     title: string;
     uri: string;
-    headingPath: string[];
     snippet: string;
     text?: string;
     publicationDate?: string | null;
@@ -249,7 +250,6 @@ export type ContextSource = {
     sourceId: string;
     title: string;
     uri: string;
-    headingPath: string[];
     text: string;
     metadata: Record<string, unknown>;
 };

package/dist/vector/dense.d.ts

@@ -1,9 +1,11 @@
+import { type ProgressHandler } from "../core/progress.js";
 import type { DenseVectorPayload, WorkspaceConfig } from "../types/models.js";
 export declare function setDenseEmbedderFactoryForTests(factory: ((cacheDir: string, modelId: string) => Promise<(text: string) => Promise<number[]>>) | null): void;
 export declare function pullDenseModel(workspacePath: string, config: WorkspaceConfig["retrieval"]["dense"]): Promise<void>;
-export declare function buildDenseVectors({ workspacePath, config }: {
+export declare function buildDenseVectors({ workspacePath, config, progress }: {
     workspacePath: string;
     config: WorkspaceConfig["retrieval"]["dense"];
+    progress?: ProgressHandler;
 }): Promise<DenseVectorPayload>;
 export declare function denseQuery({ workspacePath, config, query, topK }: {
     workspacePath: string;

package/dist/vector/runtime.d.ts

@@ -1,8 +1,10 @@
 import type { SparseVectorModelConfig } from "../types/models.js";
+export declare function resolveQliHomeDir(): string;
 export declare function resolveCacheDir(workspacePath: string, configuredPath: string): string;
 export declare function packageRootFromImportMeta(importMetaUrl: string): string;
 export declare function sparseScriptPath(importMetaUrl: string): Promise<string>;
 export declare function ensureUvAvailable(): Promise<void>;
+export declare function isUvAvailable(): Promise<boolean>;
 export declare function runSparsePython({ workspacePath, config, payload, importMetaUrl }: {
     workspacePath: string;
     config: SparseVectorModelConfig;

package/dist/vector/service.d.ts

@@ -1,4 +1,13 @@
 import type { DenseVectorPayload, SparseVectorPayload, WorkspaceConfig } from "../types/models.js";
+import { type ProgressHandler } from "../core/progress.js";
+import { buildModelStatus } from "./store.js";
+export declare function setPullModelsForTests(override: ((args: {
+    workspacePath: string;
+    config: WorkspaceConfig;
+    pullDense: boolean;
+    pullSparse: boolean;
+    progress?: ProgressHandler;
+}) => Promise<void>) | null): void;
 export declare function resolveModelPullPlan({ pullDenseFlag, pullSparseFlag, uvAvailable }: {
     pullDenseFlag: boolean;
     pullSparseFlag: boolean;
@@ -7,20 +16,29 @@ export declare function resolveModelPullPlan({ pullDenseFlag, pullSparseFlag, uv
     pullDense: boolean;
     pullSparse: boolean;
 };
-export declare function buildVectorArtifacts({ workspacePath, config, denseOverride, sparseOverride, buildAvailableModels }: {
+export declare function resolveMissingConfiguredModelPullPlan({ config, status }: {
+    config: WorkspaceConfig;
+    status: Awaited<ReturnType<typeof buildModelStatus>>;
+}): {
+    pullDense: boolean;
+    pullSparse: boolean;
+};
+export declare function buildVectorArtifacts({ workspacePath, config, denseOverride, sparseOverride, buildAvailableModels, progress }: {
     workspacePath: string;
     config: WorkspaceConfig;
     denseOverride?: boolean;
     sparseOverride?: boolean;
     buildAvailableModels?: boolean;
+    progress?: ProgressHandler;
 }): Promise<{
     dense?: DenseVectorPayload;
     sparse?: SparseVectorPayload;
 }>;
-export declare function pullModels({ workspacePath, config, pullDense, pullSparse }: {
+export declare function pullModels({ workspacePath, config, pullDense, pullSparse, progress }: {
     workspacePath: string;
     config: WorkspaceConfig;
     pullDense: boolean;
     pullSparse: boolean;
+    progress?: ProgressHandler;
 }): Promise<void>;
 export declare function getModelStatus(workspacePath: string, config: WorkspaceConfig): Promise<import("../index.js").ModelStatusResponse>;

package/dist/vector/sparse.d.ts

@@ -1,4 +1,5 @@
 import { type SparseVector } from "@tryformation/querylight-ts";
+import { type ProgressHandler } from "../core/progress.js";
 import type { ChunkRecord, SparseVectorPayload, SparseVectorRecord, WorkspaceConfig } from "../types/models.js";
 export declare function setSparseQueryEncoderFactoryForTests(factory: ((cacheDir: string, modelId: string, queryTokenWeights: number[]) => Promise<(text: string) => Promise<SparseVector>>) | null): void;
 export declare function setSparseDocumentBuilderFactoryForTests(factory: ((workspacePath: string, config: WorkspaceConfig["retrieval"]["sparse"], chunks: ChunkRecord[]) => Promise<{
@@ -7,9 +8,10 @@ export declare function setSparseDocumentBuilderFactoryForTests(factory: ((works
     chunks: SparseVectorRecord[];
 }>) | null): void;
 export declare function pullSparseModel(workspacePath: string, config: WorkspaceConfig["retrieval"]["sparse"]): Promise<void>;
-export declare function buildSparseVectors({ workspacePath, config }: {
+export declare function buildSparseVectors({ workspacePath, config, progress }: {
     workspacePath: string;
     config: WorkspaceConfig["retrieval"]["sparse"];
+    progress?: ProgressHandler;
 }): Promise<SparseVectorPayload>;
 export declare function sparseQuery({ workspacePath, config, query, topK }: {
     workspacePath: string;

package/dist/vector/store.d.ts

@@ -7,8 +7,14 @@ export declare function writeDensePayload(workspacePath: string, payload: DenseV
 export declare function readDensePayload(workspacePath: string): Promise<DenseVectorPayload>;
 export declare function writeSparsePayload(workspacePath: string, payload: SparseVectorPayload): Promise<void>;
 export declare function readSparsePayload(workspacePath: string): Promise<SparseVectorPayload>;
-export declare function writeDensePullMarker(workspacePath: string, value: object): Promise<void>;
-export declare function writeSparsePullMarker(workspacePath: string, value: object): Promise<void>;
+export declare function writeDensePullMarker(workspacePath: string, model: {
+    modelId: string;
+    cacheDir: string;
+}, value: object): Promise<void>;
+export declare function writeSparsePullMarker(workspacePath: string, model: {
+    modelId: string;
+    cacheDir: string;
+}, value: object): Promise<void>;
 export declare function buildModelStatus(workspacePath: string, dense: {
     enabled: boolean;
     modelId: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tryformation/querylight-cli",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "Querylight CLI for building and querying local knowledge bases.",
   "license": "MIT",
   "homepage": "https://github.com/formation-res/querylight-cli#readme",