@autorag/librarian 2.4.2 → 2.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/NOTICE +68 -0
  2. package/README.md +77 -32
  3. package/dist/agent/agent.d.ts +73 -1
  4. package/dist/agent/fast-answer-tool.d.ts +46 -0
  5. package/dist/agent/peer-target-tool.d.ts +19 -0
  6. package/dist/agent/search-all-tool.d.ts +3 -0
  7. package/dist/agent/search-datasource-tool.d.ts +3 -0
  8. package/dist/agent/search-documents.d.ts +50 -5
  9. package/dist/agent/search-minsync-tool.d.ts +3 -0
  10. package/dist/agent/system-prompt.d.ts +1 -0
  11. package/dist/cli/command-help.d.ts +1 -0
  12. package/dist/cli/commands/gateway.d.ts +7 -0
  13. package/dist/cli/commands/lite-retrieve.d.ts +9 -0
  14. package/dist/cli/commands/models.d.ts +18 -0
  15. package/dist/cli/commands/p2p-policy.d.ts +2 -0
  16. package/dist/cli/commands/p2p.d.ts +7 -0
  17. package/dist/cli/commands/serve.d.ts +18 -0
  18. package/dist/cli/commands/setup.d.ts +2 -0
  19. package/dist/cli/commands/tui.d.ts +1 -1
  20. package/dist/cli/config.d.ts +20 -0
  21. package/dist/cli/index.d.ts +3 -1
  22. package/dist/cli/index.js +13892 -8650
  23. package/dist/cli/output.d.ts +6 -0
  24. package/dist/cli/setup.d.ts +61 -0
  25. package/dist/core.js +4965 -1385
  26. package/dist/datasource/connector-skill.d.ts +1 -1
  27. package/dist/datasource/connector.d.ts +1 -1
  28. package/dist/datasource/index.d.ts +1 -2
  29. package/dist/datasource/skills/discrawl/client.d.ts +12 -2
  30. package/dist/datasource/skills/discrawl/index.d.ts +1 -1
  31. package/dist/datasource/skills/discrawl/skill.d.ts +2 -1
  32. package/dist/datasource/skills/discrawl/types.d.ts +29 -21
  33. package/dist/datasource/skills/factory.d.ts +2 -1
  34. package/dist/datasource/skills/katok/client.d.ts +4 -7
  35. package/dist/datasource/skills/katok/types.d.ts +1 -6
  36. package/dist/embedding-runtime/cache.d.ts +21 -0
  37. package/dist/embedding-runtime/gateway.d.ts +27 -0
  38. package/dist/embedding-runtime/index.d.ts +120 -0
  39. package/dist/embedding-runtime/manifest.d.ts +24 -0
  40. package/dist/embedding-runtime/supervisor.d.ts +74 -0
  41. package/dist/embedding-runtime/types.d.ts +66 -0
  42. package/dist/filesystem/source-paths.d.ts +33 -2
  43. package/dist/index.js +4974 -1389
  44. package/dist/minsync/client.d.ts +25 -1
  45. package/dist/minsync/embedder-config.d.ts +15 -0
  46. package/dist/minsync/index.d.ts +4 -3
  47. package/dist/minsync/installer.d.ts +1 -1
  48. package/dist/minsync/local-embedder.d.ts +16 -0
  49. package/dist/minsync/method.d.ts +5 -0
  50. package/dist/minsync/types.d.ts +15 -0
  51. package/dist/minsync/workspace.d.ts +6 -0
  52. package/dist/mirror/index-store.d.ts +17 -0
  53. package/dist/mirror/sync.d.ts +8 -4
  54. package/dist/observability/run-log.d.ts +2 -0
  55. package/dist/p2p/approval-store.d.ts +31 -0
  56. package/dist/p2p/egress-gate.d.ts +22 -0
  57. package/dist/p2p/file-sharing.d.ts +33 -0
  58. package/dist/p2p/injection-classifier.d.ts +33 -0
  59. package/dist/p2p/injection-gate.d.ts +37 -0
  60. package/dist/p2p/pii-gate.d.ts +47 -0
  61. package/dist/p2p/policy-filter.d.ts +15 -0
  62. package/dist/p2p/policy.d.ts +77 -0
  63. package/dist/p2p/simplex-server.d.ts +95 -0
  64. package/dist/p2p/simplex-transport.d.ts +38 -0
  65. package/dist/p2p/wire.d.ts +39 -0
  66. package/dist/process/terminate-tree.d.ts +20 -0
  67. package/licenses/apache-2.0.txt +202 -0
  68. package/licenses/embedding-assets.json +104 -0
  69. package/licenses/gemma-notice.txt +24 -0
  70. package/licenses/gemma-section-3.2-flow-down.txt +9 -0
  71. package/licenses/gemma-terms-of-use.html +4750 -0
  72. package/licenses/gemma-terms-of-use.txt +158 -0
  73. package/licenses/llama.cpp-MIT.txt +21 -0
  74. package/licenses/qwen3-embedding-notice.txt +13 -0
  75. package/package.json +15 -10
  76. package/skills/autorag/SKILL.md +18 -6
  77. package/skills/autorag-lite-search/SKILL.md +24 -6
  78. package/skills/autorag-lite-setup/SKILL.md +10 -2
  79. package/skills/autorag-setup/SKILL.md +7 -9
  80. package/dist/datasource/skills/gmail/connector.d.ts +0 -30
  81. package/dist/datasource/skills/gmail/index.d.ts +0 -2
  82. package/dist/datasource/skills/gmail/skill.d.ts +0 -24
@@ -1,6 +1,6 @@
1
1
  /**
2
2
  * Shared {@link DatasourceSkill} implementation for connector-backed
3
- * datasources (Slack, Discord, Notion, GitHub, Google Drive, Gmail/IMAP,
3
+ * datasources (Slack, Discord, Notion, GitHub, Google Drive,
4
4
  * local mail exports, RSS/news).
5
5
  *
6
6
  * Composition: a trusted {@link DatasourceConnector} fetches documents, a
@@ -1,6 +1,6 @@
1
1
  /**
2
2
  * Connector contract shared by the built-in datasource skills (Slack, Discord,
3
- * Notion, GitHub, Google Drive, Gmail/IMAP, local mail exports,
3
+ * Notion, GitHub, Google Drive, local mail exports,
4
4
  * RSS/news).
5
5
  *
6
6
  * A *connector* is the trusted, server-configured bridge to one external
@@ -9,10 +9,9 @@ export { DatasourceSkillRegistry, type RegisteredDatasourceSkill } from "./regis
9
9
  export { DatasourceResultFilter, type ResultsByMethod } from "./result-filter.js";
10
10
  export { buildDatasourceChunkSource, buildDatasourceInstanceSource, DATASOURCE_CHUNKS_SEGMENT, datasourceSourceHasFragment, datasourceSourcePath, isDatasourceSource, matchesDatasourceScope, matchesVirtualPathScope, normalizeVirtualPath, } from "./scope.js";
11
11
  export { CLOUD_DRIVE_SKILL_DEFINITION, CloudDriveSkill, type CloudDriveSkillOptions, RcloneConnector, type RcloneConnectorOptions, type RcloneRunner, type RcloneRunResult, } from "./skills/cloud-drive/index.js";
12
- export { DEFAULT_DISCRAWL_EMBEDDING_MODEL, DEFAULT_DISCRAWL_MODE, DEFAULT_DISCRAWL_SOURCE, DiscrawlClient, type DiscrawlFailureReason, DiscrawlFtsMethod, DiscrawlHybridMethod, type DiscrawlOptions, type DiscrawlSearchHit, type DiscrawlSearchMode, type DiscrawlSearchOptions, type DiscrawlSearchResult, DiscrawlSemanticMethod, DiscrawlSkill, type DiscrawlSkillClient, type DiscrawlSkillOptions, type DiscrawlSourceKind, ENGLISH_ONLY_EMBEDDING_MODELS, } from "./skills/discrawl/index.js";
12
+ export { DEFAULT_DISCRAWL_EMBEDDING_MODEL, DEFAULT_DISCRAWL_MODE, DiscrawlClient, type DiscrawlFailureReason, DiscrawlFtsMethod, DiscrawlHybridMethod, type DiscrawlOptions, type DiscrawlSearchHit, type DiscrawlSearchMode, type DiscrawlSearchOptions, type DiscrawlSearchResult, DiscrawlSemanticMethod, DiscrawlSkill, type DiscrawlSkillClient, type DiscrawlSkillOptions, ENGLISH_ONLY_EMBEDDING_MODELS, } from "./skills/discrawl/index.js";
13
13
  export { BUILTIN_DATASOURCE_SKILL_NAMES, type BuildDatasourceSkillsResult, buildDatasourceSkills, type DatasourceSkillConfig, type DatasourcesConfig, } from "./skills/factory.js";
14
14
  export { GitHubConnector, type GitHubConnectorOptions, GitHubSkill, type GitHubSkillOptions, } from "./skills/github/index.js";
15
- export { GmailConnector, type GmailConnectorOptions, GmailSkill, type GmailSkillOptions, } from "./skills/gmail/index.js";
16
15
  export type { KatokFailureReason, KatokHit, KatokOptions, KatokSearchMode, KatokSearchOptions, KatokSearchResult, KatokSkillClient, KatokSkillOptions, } from "./skills/katok/index.js";
17
16
  export { KatokBm25Method, KatokClient, KatokSemanticMethod, KatokSkill, } from "./skills/katok/index.js";
18
17
  export { MailExportConnector, type MailExportConnectorOptions, MailExportSkill, type MailExportSkillOptions, } from "./skills/mail-export/index.js";
@@ -1,4 +1,4 @@
1
- import type { DiscrawlDoctorResult, DiscrawlEmbedResult, DiscrawlOptions, DiscrawlSearchMode, DiscrawlSearchOptions, DiscrawlSearchResult, DiscrawlStatusResult, DiscrawlSyncResult } from "./types.js";
1
+ import type { DiscrawlDoctorResult, DiscrawlEmbeddingConfigResult, DiscrawlEmbeddingRuntime, DiscrawlEmbedResult, DiscrawlMetadataResult, DiscrawlOptions, DiscrawlSearchMode, DiscrawlSearchOptions, DiscrawlSearchResult, DiscrawlStatusResult, DiscrawlSyncResult } from "./types.js";
2
2
  /**
3
3
  * Thin external `discrawl` CLI wrapper. Every method spawns the `discrawl`
4
4
  * binary as a child process, parses JSON from stdout, and returns a
@@ -12,12 +12,22 @@ import type { DiscrawlDoctorResult, DiscrawlEmbedResult, DiscrawlOptions, Discra
12
12
  export declare class DiscrawlClient {
13
13
  private readonly options;
14
14
  constructor(options?: DiscrawlOptions);
15
+ configureEmbeddings(runtime: DiscrawlEmbeddingRuntime): Promise<DiscrawlEmbeddingConfigResult>;
16
+ prepareEmbeddings(): Promise<boolean>;
17
+ metadata(signal?: AbortSignal): Promise<DiscrawlMetadataResult>;
15
18
  doctor(signal?: AbortSignal): Promise<DiscrawlDoctorResult>;
16
19
  status(signal?: AbortSignal): Promise<DiscrawlStatusResult>;
20
+ /**
21
+ * Imports new messages from the local Discord Desktop cache through
22
+ * `discrawl wiretap`. AutoRAG never drives discrawl's bot sync — that path
23
+ * belongs to the CLI, which owns its own credential — so no Discord token is
24
+ * involved in an AutoRAG refresh.
25
+ */
17
26
  sync(signal?: AbortSignal): Promise<DiscrawlSyncResult>;
18
- embed(limit?: number, signal?: AbortSignal): Promise<DiscrawlEmbedResult>;
27
+ embed(limit?: number, rebuild?: boolean, signal?: AbortSignal): Promise<DiscrawlEmbedResult>;
19
28
  search(mode: DiscrawlSearchMode, query: string, options?: DiscrawlSearchOptions): Promise<DiscrawlSearchResult>;
20
29
  private runJson;
21
30
  private run;
22
31
  }
23
32
  export declare function discrawlWorkspace(options: DiscrawlOptions): string | undefined;
33
+ export declare function nativeGatewayBaseUrl(baseUrl: string): string;
@@ -2,4 +2,4 @@ export { DiscrawlClient, discrawlWorkspace } from "./client.js";
2
2
  export { DiscrawlFtsMethod, DiscrawlHybridMethod, type DiscrawlMethodOptions, type DiscrawlSearchClient, DiscrawlSemanticMethod, } from "./methods.js";
3
3
  export { DISCRAWL_SOURCE_KIND, discrawlConfigPath, discrawlDatabasePath, discrawlDatasourceRoot, discrawlSourcePath, parseDiscrawlSourcePath, } from "./paths.js";
4
4
  export { DiscrawlSkill, type DiscrawlSkillClient, type DiscrawlSkillOptions } from "./skill.js";
5
- export { DEFAULT_DISCRAWL_BINARY, DEFAULT_DISCRAWL_EMBEDDING_MODEL, DEFAULT_DISCRAWL_EMBEDDING_PROVIDER, DEFAULT_DISCRAWL_MAX_BUFFER_BYTES, DEFAULT_DISCRAWL_MODE, DEFAULT_DISCRAWL_SOURCE, DEFAULT_DISCRAWL_TIMEOUT_MS, type DiscrawlDoctorInfo, type DiscrawlDoctorResult, type DiscrawlEmbedInfo, type DiscrawlEmbedResult, type DiscrawlFailure, type DiscrawlFailureReason, type DiscrawlMessage, type DiscrawlOptions, type DiscrawlSearchHit, type DiscrawlSearchMode, type DiscrawlSearchOptions, type DiscrawlSearchResult, type DiscrawlSourceKind, type DiscrawlStatusInfo, type DiscrawlStatusResult, type DiscrawlSyncInfo, type DiscrawlSyncResult, ENGLISH_ONLY_EMBEDDING_MODELS, } from "./types.js";
5
+ export { DEFAULT_DISCRAWL_BINARY, DEFAULT_DISCRAWL_EMBEDDING_MODEL, DEFAULT_DISCRAWL_EMBEDDING_PROVIDER, DEFAULT_DISCRAWL_MAX_BUFFER_BYTES, DEFAULT_DISCRAWL_MODE, DEFAULT_DISCRAWL_TIMEOUT_MS, type DiscrawlDoctorInfo, type DiscrawlDoctorResult, type DiscrawlEmbeddingConfigResult, type DiscrawlEmbeddingRuntime, type DiscrawlEmbedInfo, type DiscrawlEmbedResult, type DiscrawlFailure, type DiscrawlFailureReason, type DiscrawlMessage, type DiscrawlMetadataInfo, type DiscrawlMetadataResult, type DiscrawlOptions, type DiscrawlSearchHit, type DiscrawlSearchMode, type DiscrawlSearchOptions, type DiscrawlSearchResult, type DiscrawlStatusInfo, type DiscrawlStatusResult, type DiscrawlSyncInfo, type DiscrawlSyncResult, ENGLISH_ONLY_EMBEDDING_MODELS, } from "./types.js";
@@ -5,7 +5,8 @@ import type { DiscrawlDoctorResult, DiscrawlEmbedResult, DiscrawlSearchMode, Dis
5
5
  export interface DiscrawlSkillClient extends DiscrawlSearchClient {
6
6
  doctor(): Promise<DiscrawlDoctorResult>;
7
7
  sync(): Promise<DiscrawlSyncResult>;
8
- embed(limit?: number): Promise<DiscrawlEmbedResult>;
8
+ embed(limit?: number, rebuild?: boolean): Promise<DiscrawlEmbedResult>;
9
+ prepareEmbeddings?(): Promise<boolean>;
9
10
  }
10
11
  export interface DiscrawlSkillOptions {
11
12
  readonly client: DiscrawlSkillClient;
@@ -8,22 +8,6 @@ import type { RetrievalOptions } from "../../../retrieval/types.js";
8
8
  * semantic is unavailable.
9
9
  */
10
10
  export type DiscrawlSearchMode = "fts" | "semantic" | "hybrid";
11
- /**
12
- * Where the `discrawl` CLI reads Discord data from.
13
- *
14
- * - `wiretap` — the local Discord Desktop cache. Requires no token and issues
15
- * no Discord API calls; it reads only files Discord already
16
- * wrote to disk. Includes classifiable direct messages.
17
- * - `discord` — the Discord bot API using a server-configured bot token. This
18
- * is the ToS-sanctioned automation path (a bot token identifies
19
- * an OAuth2 application, not a human user).
20
- * - `both` — bot API plus desktop cache.
21
- *
22
- * AutoRAG never uses a Discord *user* token: automating a user account is
23
- * prohibited by Discord's Community Guidelines and can get the account
24
- * terminated.
25
- */
26
- export type DiscrawlSourceKind = "wiretap" | "discord" | "both";
27
11
  /**
28
12
  * Embedding models known to be English-only. Selecting one for a non-English
29
13
  * archive silently degrades semantic search to noise rather than failing, so
@@ -49,7 +33,31 @@ export declare const DEFAULT_DISCRAWL_EMBEDDING_PROVIDER = "ollama";
49
33
  * Configuration for the discrawl client. All fields optional; defaults mirror
50
34
  * the katok client. The client spawns the `discrawl` binary as a child process
51
35
  * — it never opens the Discord archive database directly.
36
+ *
37
+ * AutoRAG drives discrawl's local archive only: `wiretap` to import the
38
+ * Discord Desktop cache plus search/embed against that native store. The bot
39
+ * API path belongs to the CLI, which owns its own credential, so no Discord
40
+ * token is configured, required, or forwarded here.
52
41
  */
42
+ export interface DiscrawlEmbeddingRuntime {
43
+ readonly provider: string;
44
+ readonly model: string;
45
+ readonly baseUrl: string;
46
+ readonly dimensions: number;
47
+ readonly identity?: string;
48
+ }
49
+ export interface DiscrawlEmbeddingConfigResult {
50
+ readonly configured: boolean;
51
+ readonly rebuildRequired: boolean;
52
+ readonly diagnostic?: string;
53
+ }
54
+ export interface DiscrawlMetadataInfo {
55
+ readonly embeddingProvider?: string;
56
+ readonly embeddingModel?: string;
57
+ readonly embeddingDimensions?: number;
58
+ readonly embeddingIdentity?: string;
59
+ readonly metadata?: Readonly<Record<string, unknown>>;
60
+ }
53
61
  export interface DiscrawlOptions {
54
62
  /** Explicit path to the `discrawl` binary. Defaults to a bare PATH lookup. */
55
63
  readonly binaryPath?: string;
@@ -57,8 +65,6 @@ export interface DiscrawlOptions {
57
65
  readonly timeoutMs?: number;
58
66
  /** Max stdout/stderr bytes retained. Default 1_048_576 (1 MiB). */
59
67
  readonly maxBufferBytes?: number;
60
- /** Archive source for the CLI. Default `wiretap` (no token required). */
61
- readonly source?: DiscrawlSourceKind;
62
68
  /**
63
69
  * Explicit operator-owned discrawl config file passed as `--config`.
64
70
  * AutoRAG never rewrites an explicit config.
@@ -68,10 +74,12 @@ export interface DiscrawlOptions {
68
74
  readonly root?: string;
69
75
  /** Explicit workspace directory overriding the computed default. */
70
76
  readonly workspacePath?: string;
71
- /** Restrict search and sync to one guild id. */
77
+ /** Restrict search to one guild id. */
72
78
  readonly guildId?: string;
73
79
  /** Embedding provider understood by the native discrawl configuration. */
74
80
  readonly embeddingProvider?: string;
81
+ /** AutoRAG-owned runtime to write into a managed workspace config. */
82
+ readonly embeddingRuntime?: DiscrawlEmbeddingRuntime;
75
83
  /**
76
84
  * Embedding model understood by the native discrawl configuration.
77
85
  * English-only models are accepted but reported through a diagnostic.
@@ -91,7 +99,6 @@ export interface DiscrawlOptions {
91
99
  export declare const DEFAULT_DISCRAWL_BINARY = "discrawl";
92
100
  export declare const DEFAULT_DISCRAWL_TIMEOUT_MS = 30000;
93
101
  export declare const DEFAULT_DISCRAWL_MAX_BUFFER_BYTES = 1048576;
94
- export declare const DEFAULT_DISCRAWL_SOURCE: DiscrawlSourceKind;
95
102
  export declare const DEFAULT_DISCRAWL_MODE: DiscrawlSearchMode;
96
103
  /**
97
104
  * Reasons a discrawl CLI invocation can fail. The client never throws for
@@ -147,7 +154,7 @@ export interface DiscrawlDoctorInfo {
147
154
  readonly embeddingProvider?: string;
148
155
  readonly metadata?: Readonly<Record<string, unknown>>;
149
156
  }
150
- /** Result of `discrawl sync` / `discrawl wiretap`. */
157
+ /** Result of `discrawl wiretap`. */
151
158
  export interface DiscrawlSyncInfo {
152
159
  readonly messages: number;
153
160
  readonly guilds?: number;
@@ -184,6 +191,7 @@ export interface DiscrawlSearchOk {
184
191
  }
185
192
  export type DiscrawlDoctorResult = DiscrawlOk<DiscrawlDoctorInfo> | DiscrawlFailure;
186
193
  export type DiscrawlStatusResult = DiscrawlOk<DiscrawlStatusInfo> | DiscrawlFailure;
194
+ export type DiscrawlMetadataResult = DiscrawlOk<DiscrawlMetadataInfo> | DiscrawlFailure;
187
195
  export type DiscrawlSyncResult = DiscrawlOk<DiscrawlSyncInfo> | DiscrawlFailure;
188
196
  export type DiscrawlEmbedResult = DiscrawlOk<DiscrawlEmbedInfo> | DiscrawlFailure;
189
197
  export type DiscrawlSearchResult = DiscrawlSearchOk | DiscrawlFailure;
@@ -8,6 +8,7 @@
8
8
  * and reported so setup surfaces actionable (but path-opaque) feedback.
9
9
  */
10
10
  import type { DatasourceSkill } from "../types.js";
11
+ import { type DiscrawlEmbeddingRuntime } from "./discrawl/index.js";
11
12
  /** One configured datasource entry (the trusted `datasources.<name>` value). */
12
13
  export interface DatasourceSkillConfig {
13
14
  readonly enabled?: boolean;
@@ -41,4 +42,4 @@ export declare const BUILTIN_DATASOURCE_SKILL_NAMES: readonly string[];
41
42
  * collected in `unknown` (never thrown) so callers can surface setup
42
43
  * feedback without failing agent construction.
43
44
  */
44
- export declare function buildDatasourceSkills(config: DatasourcesConfig | undefined, workspaceRoot?: string): BuildDatasourceSkillsResult;
45
+ export declare function buildDatasourceSkills(config: DatasourcesConfig | undefined, workspaceRoot?: string, embeddingRuntime?: DiscrawlEmbeddingRuntime): BuildDatasourceSkillsResult;
@@ -2,12 +2,9 @@ import type { KatokChunkResult, KatokContextResult, KatokDoctorResult, KatokInde
2
2
  /**
3
3
  * Thin external `katok` CLI wrapper. Every method spawns the `katok` binary as
4
4
  * a child process, parses JSON from stdout, and returns a discriminated ok/fail
5
- * union. No method throws for expected failures (missing binary, rejected
6
- * remote-embedding config, CLI nonzero exit, timeout, oversized output, or
7
- * invalid JSON). The client never opens the KakaoTalk database directly.
8
- *
9
- * Remote-embedding configuration is rejected before spawn, case-insensitively,
10
- * so the spawned process can never egress embeddings to a remote endpoint.
5
+ * union. No method throws for expected failures (missing binary, CLI nonzero
6
+ * exit, timeout, oversized output, or invalid JSON). The client never opens the
7
+ * KakaoTalk database directly.
11
8
  */
12
9
  export declare class KatokClient {
13
10
  private readonly options;
@@ -19,6 +16,6 @@ export declare class KatokClient {
19
16
  chunkGet(chunkId: string, signal?: AbortSignal): Promise<KatokChunkResult>;
20
17
  context(chunkId: string, signal?: AbortSignal): Promise<KatokContextResult>;
21
18
  parent(chunkId: string, signal?: AbortSignal): Promise<KatokParentResult>;
22
- /** Single retrieval pipeline: build env, gate remote embeddings, spawn, parse-free raw result. */
19
+ /** Single retrieval pipeline: build env, spawn, parse-free raw result. */
23
20
  private run;
24
21
  }
@@ -39,7 +39,7 @@ export declare const DEFAULT_KATOK_OPTIONS: {
39
39
  * every method returns a discriminated union with `ok: false` and one of these
40
40
  * reasons.
41
41
  */
42
- export type KatokFailureReason = "binary-missing" | "remote-embedding-rejected" | "nonzero-exit" | "spawn-error" | "timeout" | "aborted" | "stdout-too-large" | "stderr-too-large" | "invalid-json" | "invalid-shape";
42
+ export type KatokFailureReason = "binary-missing" | "nonzero-exit" | "spawn-error" | "timeout" | "aborted" | "stdout-too-large" | "stderr-too-large" | "invalid-json" | "invalid-shape";
43
43
  /** Common failure payload shared by every method result union. */
44
44
  export interface KatokFailure {
45
45
  readonly ok: false;
@@ -50,11 +50,6 @@ export interface KatokFailure {
50
50
  readonly stderr: string;
51
51
  /** Exit code, or `null` when the process never exited normally. */
52
52
  readonly code: number | null;
53
- /**
54
- * The offending environment key when `reason === "remote-embedding-rejected"`.
55
- * A config key name (never a path, never the key's value).
56
- */
57
- readonly violatingKey?: string;
58
53
  readonly hits?: readonly unknown[];
59
54
  }
60
55
  /** A single indexed chat chunk returned by the katok CLI. */
@@ -0,0 +1,21 @@
1
+ export { CacheError } from "./types.js";
2
+ export interface CacheAsset {
3
+ readonly id: string;
4
+ readonly url: string;
5
+ readonly filename: string;
6
+ readonly sha256: string;
7
+ readonly kind?: "model" | "runtime";
8
+ readonly archiveMembers?: readonly string[];
9
+ }
10
+ export declare function cacheDirectory(kind: "models" | "runtime", cacheRoot?: string): string;
11
+ export interface CacheOptions {
12
+ readonly cacheRoot?: string;
13
+ readonly offline?: boolean;
14
+ readonly fetch?: typeof globalThis.fetch;
15
+ /** Total budget (ms) for a single asset download. Defaults to 10 minutes so large models can transfer. */
16
+ readonly downloadTimeoutMs?: number;
17
+ }
18
+ export declare function verifyCacheEntry(path: string, expectedSha256: string): Promise<string>;
19
+ export declare function hasVerifiedAsset(asset: CacheAsset, cacheRoot?: string): Promise<boolean>;
20
+ export declare function downloadAsset(asset: CacheAsset, options?: CacheOptions): Promise<string>;
21
+ export declare function importAsset(sourcePath: string, asset: CacheAsset, options?: CacheOptions): Promise<string>;
@@ -0,0 +1,27 @@
1
+ import type { RuntimeProfile } from "./types.js";
2
+ export type GatewayErrorCode = "non-loopback-host" | "non-loopback-upstream" | "bad-request" | "batch-limit" | "upstream" | "upstream-violation" | "timeout";
3
+ export declare class GatewayError extends Error {
4
+ readonly code: GatewayErrorCode;
5
+ readonly status: number;
6
+ constructor(code: GatewayErrorCode, message: string, status?: number, options?: {
7
+ readonly cause?: unknown;
8
+ });
9
+ }
10
+ export interface StartEmbeddingGatewayOptions {
11
+ readonly profile: RuntimeProfile;
12
+ readonly host?: string;
13
+ readonly port?: number;
14
+ readonly upstreamUrl: string;
15
+ readonly fetch?: typeof fetch;
16
+ readonly maxInputs?: number;
17
+ readonly maxCharacters?: number;
18
+ readonly timeoutMs?: number;
19
+ }
20
+ export interface EmbeddingGateway {
21
+ readonly host: string;
22
+ readonly port: number;
23
+ readonly url: string;
24
+ close(): Promise<void>;
25
+ }
26
+ export declare function isLoopbackHost(host: string): boolean;
27
+ export declare function startEmbeddingGateway(options: StartEmbeddingGatewayOptions): Promise<EmbeddingGateway>;
@@ -0,0 +1,120 @@
1
+ import { type CacheOptions } from "./cache.js";
2
+ import { startEmbeddingGateway } from "./gateway.js";
3
+ import { type SupervisorStatus } from "./supervisor.js";
4
+ import type { BackendKind, HealthStatus, ProfileId, RuntimeProfile } from "./types.js";
5
+ export interface EmbeddingRuntimeCache {
6
+ downloadAsset(asset: {
7
+ id: string;
8
+ url: string;
9
+ filename: string;
10
+ sha256: string;
11
+ kind?: "model" | "runtime";
12
+ archiveMembers?: readonly string[];
13
+ }, options?: CacheOptions): Promise<string>;
14
+ importAsset(sourcePath: string, asset: {
15
+ id: string;
16
+ url: string;
17
+ filename: string;
18
+ sha256: string;
19
+ kind?: "model" | "runtime";
20
+ archiveMembers?: readonly string[];
21
+ }, options?: CacheOptions): Promise<string>;
22
+ verifyCacheEntry(path: string, expectedSha256: string): Promise<string>;
23
+ hasVerifiedAsset?(asset: {
24
+ id: string;
25
+ url: string;
26
+ filename: string;
27
+ sha256: string;
28
+ kind?: "model" | "runtime";
29
+ archiveMembers?: readonly string[];
30
+ }, cacheRoot?: string): Promise<boolean>;
31
+ }
32
+ export interface EmbeddingRuntimeSupervisor {
33
+ ensureRunning(): Promise<SupervisorStatus>;
34
+ shutdown(): Promise<void>;
35
+ status(): SupervisorStatus;
36
+ }
37
+ export interface EmbeddingRuntimeGateway {
38
+ readonly url: string;
39
+ close(): Promise<void>;
40
+ }
41
+ export interface EmbeddingRuntimeOptions {
42
+ cacheRoot?: string;
43
+ backend?: BackendKind;
44
+ offline?: boolean;
45
+ platform?: "darwin-arm64-metal" | "win-x64-cpu" | "win-x64-vulkan";
46
+ fetch?: typeof fetch;
47
+ supervisor?: EmbeddingRuntimeSupervisor;
48
+ cache?: EmbeddingRuntimeCache;
49
+ gatewayFactory?: (options: Parameters<typeof startEmbeddingGateway>[0]) => Promise<EmbeddingRuntimeGateway>;
50
+ }
51
+ export interface EnsureRuntimeOptions extends EmbeddingRuntimeOptions {
52
+ profileId?: ProfileId;
53
+ /** Refuse downloads; used by MinSync refresh/query paths. */
54
+ cachedOnly?: boolean;
55
+ }
56
+ export interface RuntimeIdentity {
57
+ readonly profileId: ProfileId;
58
+ readonly provider: string;
59
+ readonly model: string;
60
+ readonly dimension: number;
61
+ readonly runtimeBuild: string;
62
+ readonly modelRevision: string;
63
+ readonly artifactSha256: string;
64
+ }
65
+ export interface EnsuredRuntime {
66
+ readonly baseUrl: string;
67
+ readonly profile: RuntimeProfile;
68
+ readonly identity: RuntimeIdentity;
69
+ readonly supervisor: SupervisorStatus;
70
+ }
71
+ export interface RuntimeStatus {
72
+ readonly state: SupervisorStatus["state"];
73
+ readonly backend: BackendKind;
74
+ readonly model: string;
75
+ readonly pid?: number;
76
+ readonly port?: number;
77
+ readonly uptimeMs?: number;
78
+ readonly lastError?: string;
79
+ readonly profileId?: ProfileId;
80
+ readonly baseUrl?: string;
81
+ readonly health: HealthStatus;
82
+ }
83
+ export declare function createEmbeddingRuntime(options?: EmbeddingRuntimeOptions): {
84
+ resolveRuntimeProfile: (profileId?: ProfileId) => RuntimeProfile;
85
+ ensureRuntime: (input?: EnsureRuntimeOptions) => Promise<EnsuredRuntime>;
86
+ releaseRuntimeHandles: () => Promise<void>;
87
+ stopRuntime: () => Promise<void>;
88
+ runtimeStatus: () => Promise<RuntimeStatus>;
89
+ prefetchModel: (profileId?: ProfileId) => Promise<{
90
+ profileId: ProfileId;
91
+ path: string;
92
+ }>;
93
+ importModel: (profileId: ProfileId | undefined, sourcePath: string) => Promise<{
94
+ profileId: ProfileId;
95
+ path: string;
96
+ }>;
97
+ verifyModel: (profileId?: ProfileId) => Promise<{
98
+ profileId: ProfileId;
99
+ path: string;
100
+ hash: string;
101
+ }>;
102
+ };
103
+ export declare const resolveRuntimeProfile: (profileId?: ProfileId) => RuntimeProfile;
104
+ export declare const ensureRuntime: (input?: EnsureRuntimeOptions) => Promise<EnsuredRuntime>;
105
+ export declare const releaseRuntimeHandles: () => Promise<void>;
106
+ export declare const stopRuntime: () => Promise<void>;
107
+ export declare const runtimeStatus: () => Promise<RuntimeStatus>;
108
+ export declare const prefetchModel: (profileId?: ProfileId) => Promise<{
109
+ profileId: ProfileId;
110
+ path: string;
111
+ }>;
112
+ export declare const importModel: (profileId: ProfileId | undefined, sourcePath: string) => Promise<{
113
+ profileId: ProfileId;
114
+ path: string;
115
+ }>;
116
+ export declare const verifyModel: (profileId?: ProfileId) => Promise<{
117
+ profileId: ProfileId;
118
+ path: string;
119
+ hash: string;
120
+ }>;
@@ -0,0 +1,24 @@
1
+ import type { BackendKind, PlatformId, ProfileId, RuntimeAsset, RuntimeProfile } from "./types.js";
2
+ export declare function resolveProfile(profileId: ProfileId): RuntimeProfile;
3
+ export declare function platformRuntimeAsset(platform: PlatformId): RuntimeAsset;
4
+ export declare function selectPlatformAsset(platform: PlatformId, backend: BackendKind, vulkanAvailable?: boolean): RuntimeAsset;
5
+ export declare const MODEL_ASSETS: {
6
+ readonly qwen3: {
7
+ readonly url: "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B-GGUF/resolve/370f27d7550e0def9b39c1f16d3fbaa13aa67728/Qwen3-Embedding-0.6B-Q8_0.gguf";
8
+ readonly revision: "370f27d7550e0def9b39c1f16d3fbaa13aa67728";
9
+ readonly sha256: "06507c7b42688469c4e7298b0a1e16deff06caf291cf0a5b278c308249c3e439";
10
+ readonly license: "Apache-2.0";
11
+ readonly noticeReference: "Qwen3 model card and Apache-2.0 license";
12
+ };
13
+ readonly embeddinggemma: {
14
+ readonly url: "https://huggingface.co/ggml-org/embeddinggemma-300M-GGUF/resolve/0f741b5a6585bd53aeb15cd1372c56f2a0f65e12/embeddinggemma-300M-Q8_0.gguf";
15
+ readonly revision: "0f741b5a6585bd53aeb15cd1372c56f2a0f65e12";
16
+ readonly sha256: "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63";
17
+ readonly license: "Gemma Terms of Use";
18
+ readonly noticeReference: "Google EmbeddingGemma model card and Gemma Terms of Use";
19
+ };
20
+ };
21
+ export declare const MODEL_ASSET_URLS: {
22
+ readonly qwen3: "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B-GGUF/resolve/370f27d7550e0def9b39c1f16d3fbaa13aa67728/Qwen3-Embedding-0.6B-Q8_0.gguf";
23
+ readonly embeddinggemma: "https://huggingface.co/ggml-org/embeddinggemma-300M-GGUF/resolve/0f741b5a6585bd53aeb15cd1372c56f2a0f65e12/embeddinggemma-300M-Q8_0.gguf";
24
+ };
@@ -0,0 +1,74 @@
1
+ import { type ChildProcess, type SpawnOptions } from "node:child_process";
2
+ import type { BackendKind, ProfileId } from "./types.js";
3
+ export type SupervisorState = "stopped" | "starting" | "ready" | "failed" | "stopping";
4
+ export interface SupervisorStatus {
5
+ readonly state: SupervisorState;
6
+ readonly pid?: number;
7
+ readonly port?: number;
8
+ readonly backend: BackendKind;
9
+ readonly model: string;
10
+ readonly uptimeMs?: number;
11
+ readonly lastError?: string;
12
+ }
13
+ export declare class SupervisorError extends Error {
14
+ readonly code: "lock-conflict" | "readiness-timeout" | "spawn" | "exited" | "shutdown" | "missing-executable";
15
+ constructor(code: SupervisorError["code"], message: string, options?: {
16
+ readonly cause?: unknown;
17
+ });
18
+ }
19
+ type SpawnLike = (command: string, args: readonly string[], options: SpawnOptions) => ChildProcess;
20
+ type KillLike = (pid: number, signal?: NodeJS.Signals | number) => void;
21
+ export interface SupervisorOptions {
22
+ readonly cacheRoot?: string;
23
+ readonly profileId?: ProfileId;
24
+ readonly backend?: BackendKind;
25
+ readonly executablePath?: string;
26
+ readonly modelPath?: string;
27
+ readonly readinessTimeoutMs?: number;
28
+ readonly readinessIntervalMs?: number;
29
+ readonly logMaxBytes?: number;
30
+ readonly fetch?: typeof globalThis.fetch;
31
+ readonly spawn?: SpawnLike;
32
+ readonly kill?: KillLike;
33
+ readonly platform?: NodeJS.Platform;
34
+ }
35
+ export declare const DEFAULT_READINESS_TIMEOUT_MS = 60000;
36
+ export declare class EmbeddingRuntimeSupervisor {
37
+ private readonly root;
38
+ private readonly backend;
39
+ private readonly executablePath;
40
+ private readonly modelPath;
41
+ private readonly timeoutMs;
42
+ private readonly intervalMs;
43
+ private readonly logMaxBytes;
44
+ private readonly spawnProcess;
45
+ private readonly killProcess;
46
+ private readonly fetchHealth;
47
+ private readonly osPlatform;
48
+ private readonly defaultExecutable;
49
+ private child?;
50
+ private adoptedPid?;
51
+ private startedAt?;
52
+ private currentPort?;
53
+ private state;
54
+ private lastError?;
55
+ private restartCount;
56
+ private shutdownRequested;
57
+ private stopping?;
58
+ constructor(options?: SupervisorOptions);
59
+ ensureRunning(): Promise<SupervisorStatus>;
60
+ status(): SupervisorStatus;
61
+ shutdown(): Promise<void>;
62
+ private adoptExistingRuntime;
63
+ private acquireLock;
64
+ private resolveExecutable;
65
+ private startChild;
66
+ private recoverUnexpectedExit;
67
+ private waitUntilReady;
68
+ private stopAdopted;
69
+ private stopChild;
70
+ private cleanupState;
71
+ private appendLog;
72
+ private readLog;
73
+ }
74
+ export {};
@@ -0,0 +1,66 @@
1
+ export type BackendKind = "auto" | "cpu" | "vulkan";
2
+ export type PlatformId = "darwin-arm64-metal" | "win-x64-cpu" | "win-x64-vulkan";
3
+ export type ProfileId = "qwen3-embedding-0.6b" | "embeddinggemma-300m";
4
+ export interface ProtocolCapabilities {
5
+ readonly healthz: boolean;
6
+ readonly embed: boolean;
7
+ readonly openaiCompatible: boolean;
8
+ readonly ollamaCompatible: boolean;
9
+ }
10
+ export interface RuntimeProfile {
11
+ readonly profileId: ProfileId;
12
+ readonly provider: string;
13
+ readonly model: string;
14
+ readonly dimension: number;
15
+ readonly queryPrefix: string;
16
+ readonly passagePrefix: string;
17
+ readonly runtimeBuild: string;
18
+ readonly modelRevision: string;
19
+ readonly artifactSha256: string;
20
+ readonly backend: BackendKind;
21
+ readonly protocolCapabilities: ProtocolCapabilities;
22
+ }
23
+ export interface HealthResult {
24
+ readonly ok: true;
25
+ readonly profileId: ProfileId;
26
+ readonly dimension: number;
27
+ readonly runtimeBuild: string;
28
+ }
29
+ export interface HealthFailure {
30
+ readonly ok: false;
31
+ readonly code: "unavailable" | "starting" | "incompatible" | "failed";
32
+ readonly message: string;
33
+ readonly retryable: boolean;
34
+ }
35
+ export type HealthStatus = HealthResult | HealthFailure;
36
+ export interface FailureResult {
37
+ readonly ok: false;
38
+ readonly code: string;
39
+ readonly message: string;
40
+ readonly cause?: unknown;
41
+ }
42
+ export declare class CacheError extends Error {
43
+ readonly code: "invalid-hash" | "offline-missing" | "download" | "hash-mismatch" | "io" | "extraction" | "missing-member";
44
+ readonly path?: string;
45
+ constructor(code: CacheError["code"], message: string, options?: {
46
+ readonly path?: string;
47
+ readonly cause?: unknown;
48
+ });
49
+ }
50
+ export interface ModelAsset {
51
+ readonly url: string;
52
+ readonly revision: string;
53
+ readonly sha256: string;
54
+ readonly license: string;
55
+ readonly noticeReference: string;
56
+ }
57
+ export interface RuntimeAsset {
58
+ readonly id: string;
59
+ readonly url: string;
60
+ readonly filename: string;
61
+ readonly sha256: string;
62
+ readonly version: string;
63
+ readonly platform: PlatformId;
64
+ readonly archiveMembers: readonly string[];
65
+ readonly kind: "runtime";
66
+ }
@@ -14,13 +14,44 @@ export interface ResolvedVirtualSource {
14
14
  readonly realPath: string;
15
15
  readonly sourceId: string;
16
16
  }
17
+ /**
18
+ * True when `source` is an absolute filesystem path on any supported OS
19
+ * (POSIX `/...`, Windows drive `C:\` or `C:/`, or UNC `\\...`).
20
+ * Platform-independent so peer wire ids behave identically on macOS, Linux,
21
+ * and Windows servers.
22
+ */
23
+ export declare function isFilesystemAbsolutePath(source: string): boolean;
17
24
  /**
18
25
  * Normalize an opaque virtual source id. Returns the normalized id, or
19
26
  * `undefined` when the input is syntactically invalid or unsafe (no leading
20
- * slash, URL scheme, fragment/query, traversal, NUL, backslash). Never throws
21
- * and never returns a real path — failures are signaled as `undefined`.
27
+ * slash, URL scheme, traversal, NUL, backslash). Never throws and never
28
+ * returns a real path — failures are signaled as `undefined`.
29
+ *
30
+ * `#` and `?` are NOT rejected: they are ordinary characters in a real file
31
+ * name, and ids are derived verbatim from real file names. URL-shaped input
32
+ * is already rejected by the scheme check above.
22
33
  */
23
34
  export declare function normalizeVirtualPath(virtual: string | undefined | null): string | undefined;
35
+ /**
36
+ * Canonicalize any retrieval source form to the canonical virtual source id:
37
+ *
38
+ * 1. An id already inside a configured root's virtual namespace is validated
39
+ * and returned unchanged (modulo separator cleanup).
40
+ * 2. An OS-absolute real path contained in a configured source root becomes
41
+ * that root's virtual id (`/<prefix>/<relative>`); the longest containing
42
+ * root wins for nested roots.
43
+ * 3. A datasource slash identity (`/kakao/<instance>/chunks/<chunk>`) is a
44
+ * virtual id in a non-filesystem namespace and passes through validation.
45
+ * 4. An absolute filesystem path outside every root passes through as its
46
+ * own canonical form — backslashes are converted to forward slashes so
47
+ * Windows drive/UNC paths canonicalize identically on every host — letting
48
+ * operators write policy globs against real absolute paths.
49
+ *
50
+ * Everything else fails closed as `undefined`: traversal segments, URL
51
+ * schemes (including the retired `kakao:<chat>` colon scheme), backslashes in
52
+ * non-absolute sources, and empty input.
53
+ */
54
+ export declare function normalizeSource(source: string, sourceRoots: readonly SourceRoot[]): string | undefined;
24
55
  /**
25
56
  * Exact-prefix boundary match: `/docs` matches `/docs` and `/docs/...` but NOT
26
57
  * `/docs-2`. Prevents prefix confusion between sibling roots like `/docs` and