@autorag/librarian 2.5.2 → 2.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -6
- package/dist/agent/agent.d.ts +9 -14
- package/dist/agent/index.d.ts +1 -2
- package/dist/agent/search-single-datasource-tool.d.ts +14 -8
- package/dist/cli/index.js +2130 -1057
- package/dist/core.js +1711 -665
- package/dist/datasource/errors.d.ts +1 -1
- package/dist/datasource/index.d.ts +3 -2
- package/dist/datasource/skills/discrawl/skill.d.ts +1 -1
- package/dist/datasource/skills/discrawl/types.d.ts +2 -2
- package/dist/datasource/skills/github-gist/connector.d.ts +38 -0
- package/dist/datasource/skills/github-gist/index.d.ts +3 -0
- package/dist/datasource/skills/github-gist/semantic.d.ts +102 -0
- package/dist/datasource/skills/github-gist/skill.d.ts +39 -0
- package/dist/datasource/skills/lark/client.d.ts +23 -0
- package/dist/datasource/skills/lark/index.d.ts +2 -0
- package/dist/datasource/skills/lark/skill.d.ts +36 -0
- package/dist/datasource/skills/lazykatok/client.d.ts +28 -0
- package/dist/datasource/skills/lazykatok/index.d.ts +12 -0
- package/dist/datasource/skills/{katok → lazykatok}/methods.d.ts +12 -12
- package/dist/datasource/skills/lazykatok/paths.d.ts +40 -0
- package/dist/datasource/skills/{katok → lazykatok}/skill.d.ts +10 -10
- package/dist/datasource/skills/lazykatok/types.d.ts +134 -0
- package/dist/datasource/skills/mailcrawl/client.d.ts +2 -0
- package/dist/datasource/skills/mailcrawl/types.d.ts +12 -0
- package/dist/datasource/types.d.ts +4 -4
- package/dist/embedding-runtime/manifest.d.ts +8 -0
- package/dist/embedding-runtime/types.d.ts +4 -0
- package/dist/index.js +1719 -671
- package/dist/process/terminate-tree.d.ts +2 -0
- package/dist/retrieval/types.d.ts +1 -1
- package/licenses/embedding-assets.json +20 -4
- package/package.json +1 -1
- package/skills/autorag-doctor/SKILL.md +4 -3
- package/skills/autorag-lite-setup/SKILL.md +2 -2
- package/skills/autorag-setup/SKILL.md +9 -6
- package/dist/agent/search-datasource-tool.d.ts +0 -30
- package/dist/datasource/skills/katok/client.d.ts +0 -21
- package/dist/datasource/skills/katok/index.d.ts +0 -12
- package/dist/datasource/skills/katok/paths.d.ts +0 -40
- package/dist/datasource/skills/katok/types.d.ts +0 -123
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* pipeline reports the surface as unsearched and hands the operator the CLI's
|
|
7
7
|
* own words: failure kind, exit status, and stderr verbatim.
|
|
8
8
|
*/
|
|
9
|
-
/** The failure payload every datasource CLI client returns (
|
|
9
|
+
/** The failure payload every datasource CLI client returns (lazykatok, discrawl, qmd, …). */
|
|
10
10
|
export interface DatasourceCliFailure {
|
|
11
11
|
readonly reason: string;
|
|
12
12
|
readonly stderr?: string;
|
|
@@ -12,8 +12,9 @@ export { CLOUD_DRIVE_SKILL_DEFINITION, CloudDriveSkill, type CloudDriveSkillOpti
|
|
|
12
12
|
export { DEFAULT_DISCRAWL_EMBEDDING_MODEL, DEFAULT_DISCRAWL_MODE, DiscrawlClient, type DiscrawlFailureReason, DiscrawlFtsMethod, DiscrawlHybridMethod, type DiscrawlOptions, type DiscrawlSearchHit, type DiscrawlSearchMode, type DiscrawlSearchOptions, type DiscrawlSearchResult, DiscrawlSemanticMethod, DiscrawlSkill, type DiscrawlSkillClient, type DiscrawlSkillOptions, ENGLISH_ONLY_EMBEDDING_MODELS, } from "./skills/discrawl/index.js";
|
|
13
13
|
export { BUILTIN_DATASOURCE_SKILL_NAMES, type BuildDatasourceSkillsResult, buildDatasourceSkills, type DatasourceSkillConfig, type DatasourcesConfig, } from "./skills/factory.js";
|
|
14
14
|
export { GitHubConnector, type GitHubConnectorOptions, GitHubSkill, type GitHubSkillOptions, } from "./skills/github/index.js";
|
|
15
|
-
export
|
|
16
|
-
export {
|
|
15
|
+
export { LarkClient, type LarkClientOptions, LarkSkill, type LarkSkillOptions } from "./skills/lark/index.js";
|
|
16
|
+
export type { LazykatokFailureReason, LazykatokHit, LazykatokOptions, LazykatokSearchMode, LazykatokSearchOptions, LazykatokSearchResult, LazykatokSkillClient, LazykatokSkillOptions, } from "./skills/lazykatok/index.js";
|
|
17
|
+
export { LazykatokBm25Method, LazykatokClient, LazykatokSemanticMethod, LazykatokSkill, } from "./skills/lazykatok/index.js";
|
|
17
18
|
export { MailExportConnector, type MailExportConnectorOptions, MailExportSkill, type MailExportSkillOptions, } from "./skills/mail-export/index.js";
|
|
18
19
|
export { MAILCRAWL_MODES, MailcrawlClient, type MailcrawlFailure, type MailcrawlFailureReason, type MailcrawlIndexInfo, MailcrawlMethod, type MailcrawlOptions, type MailcrawlSearchHit, type MailcrawlSearchMode, type MailcrawlSearchResult, MailcrawlSkill, type MailcrawlSyncInfo, type MailcrawlSyncResult, } from "./skills/mailcrawl/index.js";
|
|
19
20
|
export { NotcrawlClient, type NotcrawlOptions, NotionSkill, type NotionSkillOptions, } from "./skills/notion/index.js";
|
|
@@ -26,7 +26,7 @@ export interface DiscrawlSkillOptions {
|
|
|
26
26
|
/**
|
|
27
27
|
* Discord datasource skill backed by the external `discrawl` CLI.
|
|
28
28
|
*
|
|
29
|
-
* Mirrors the
|
|
29
|
+
* Mirrors the lazykatok model: the CLI owns the archive, the FTS5 index, and the
|
|
30
30
|
* vector index; AutoRAG only spawns it and maps results. AutoRAG never calls
|
|
31
31
|
* the Discord API itself and never reads the Discord Desktop cache directly.
|
|
32
32
|
*
|
|
@@ -24,14 +24,14 @@ export declare const ENGLISH_ONLY_EMBEDDING_MODELS: ReadonlySet<string>;
|
|
|
24
24
|
* Default embedding model AutoRAG configures for workspace-managed discrawl
|
|
25
25
|
* configs. `embeddinggemma`
|
|
26
26
|
* (Gemma 3 300M, 768 dimensions) covers 100+ languages and matches the
|
|
27
|
-
* EmbeddingGemma embedder
|
|
27
|
+
* EmbeddingGemma embedder lazykatok uses for KakaoTalk, so every CLI-backed
|
|
28
28
|
* datasource shares one local embedding model (served through Ollama).
|
|
29
29
|
*/
|
|
30
30
|
export declare const DEFAULT_DISCRAWL_EMBEDDING_MODEL = "embeddinggemma";
|
|
31
31
|
export declare const DEFAULT_DISCRAWL_EMBEDDING_PROVIDER = "ollama";
|
|
32
32
|
/**
|
|
33
33
|
* Configuration for the discrawl client. All fields optional; defaults mirror
|
|
34
|
-
* the
|
|
34
|
+
* the lazykatok client. The client spawns the `discrawl` binary as a child process
|
|
35
35
|
* — it never opens the Discord archive database directly.
|
|
36
36
|
*
|
|
37
37
|
* AutoRAG drives discrawl's local archive only: `wiretap` to import the
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GitHub Gist connector (issue #1588).
|
|
3
|
+
*
|
|
4
|
+
* Indexes the authenticated account's gists (public + secret, bounded by the
|
|
5
|
+
* token's scopes) through the GitHub REST API. Incremental: the gist list
|
|
6
|
+
* endpoint returns `id` + `updated_at` cheaply, so a persisted cursor map
|
|
7
|
+
* (`state.json`) decides which gists need a full content fetch and which
|
|
8
|
+
* disappeared (reported via `deletedDocIds`). Token resolution order:
|
|
9
|
+
* explicit `token` > `tokenEnv` (default `GITHUB_TOKEN`) > `gh auth token`
|
|
10
|
+
* when the gh CLI fallback is enabled. Never throws.
|
|
11
|
+
*/
|
|
12
|
+
import type { ConnectorFetchResult, DatasourceConnector } from "../../connector.js";
|
|
13
|
+
export interface GitHubGistConnectorOptions {
|
|
14
|
+
/** GitHub REST base; override to point at a mock server in tests. */
|
|
15
|
+
readonly baseUrl?: string;
|
|
16
|
+
/** Personal access token; explicit value wins over {@link tokenEnv}. */
|
|
17
|
+
readonly token?: string;
|
|
18
|
+
/** Env var name holding the token. Default `GITHUB_TOKEN`. */
|
|
19
|
+
readonly tokenEnv?: string;
|
|
20
|
+
/** When true (default), fall back to `gh auth token` for auth. */
|
|
21
|
+
readonly ghCliFallback?: boolean;
|
|
22
|
+
/** Injectable gh-token resolver (tests); defaults to spawning `gh auth token`. */
|
|
23
|
+
readonly ghTokenRunner?: () => Promise<string | undefined>;
|
|
24
|
+
readonly timeoutMs?: number;
|
|
25
|
+
readonly fetchImpl?: typeof fetch;
|
|
26
|
+
/** Persisted incremental cursor; omitted ⇒ every fetch is a full sync. */
|
|
27
|
+
readonly statePath?: string;
|
|
28
|
+
readonly maxPages?: number;
|
|
29
|
+
readonly maxGists?: number;
|
|
30
|
+
}
|
|
31
|
+
export declare class GitHubGistConnector implements DatasourceConnector {
|
|
32
|
+
private readonly options;
|
|
33
|
+
constructor(options?: GitHubGistConnectorOptions);
|
|
34
|
+
fetch(signal?: AbortSignal): Promise<ConnectorFetchResult>;
|
|
35
|
+
private resolveAuth;
|
|
36
|
+
private loadState;
|
|
37
|
+
private saveState;
|
|
38
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
export { GitHubGistConnector, type GitHubGistConnectorOptions } from "./connector.js";
|
|
2
|
+
export { createGatewayGistEmbedder, type GatewayGistEmbedderOptions, type GistEmbedder, type GistEmbeddingIdentity, GistSemanticIndex, GitHubGistSemanticMethod, } from "./semantic.js";
|
|
3
|
+
export { GITHUB_GIST_SKILL_DEFINITION, type GitHubGistSemanticOptions, GitHubGistSkill, type GitHubGistSkillOptions, gistDatasourceDir, } from "./skill.js";
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Semantic retrieval for the github-gist datasource (issue #1588).
|
|
3
|
+
*
|
|
4
|
+
* A small persisted vector sidecar (`vectors.json`) mirrors the chunk store:
|
|
5
|
+
* entries are keyed by chunk id with a content hash so unchanged chunks are
|
|
6
|
+
* never re-embedded, removed chunks are pruned, and an embedding-identity
|
|
7
|
+
* change (provider/model/dimension) forces a full re-embed. The default
|
|
8
|
+
* embedder talks to the loopback autorag-gateway (`ensureRuntime` with
|
|
9
|
+
* `cachedOnly`, then `POST /v1/embeddings`); tests inject a stub. Every
|
|
10
|
+
* failure degrades — indexing stays lexical-only and retrieval returns [].
|
|
11
|
+
*/
|
|
12
|
+
import type { ProfileId } from "../../../embedding-runtime/types.js";
|
|
13
|
+
import type { RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult } from "../../../retrieval/types.js";
|
|
14
|
+
import type { DatasourceChunkStore, StoredChunk } from "../../chunk-store.js";
|
|
15
|
+
export interface GistEmbeddingIdentity {
|
|
16
|
+
readonly provider: string;
|
|
17
|
+
readonly model: string;
|
|
18
|
+
readonly dimension: number;
|
|
19
|
+
}
|
|
20
|
+
/** Embedder contract: identity is async because the gateway resolves it lazily. */
|
|
21
|
+
export interface GistEmbedder {
|
|
22
|
+
identity(): Promise<GistEmbeddingIdentity>;
|
|
23
|
+
embed(texts: readonly string[]): Promise<readonly (readonly number[])[]>;
|
|
24
|
+
}
|
|
25
|
+
export interface GistSemanticSyncOk {
|
|
26
|
+
readonly ok: true;
|
|
27
|
+
/** Number of chunks (re-)embedded in this sync. */
|
|
28
|
+
readonly embedded: number;
|
|
29
|
+
}
|
|
30
|
+
export interface GistSemanticSyncFail {
|
|
31
|
+
readonly ok: false;
|
|
32
|
+
readonly message: string;
|
|
33
|
+
}
|
|
34
|
+
/** Persisted per-instance vector sidecar mirroring the chunk store. */
|
|
35
|
+
export declare class GistSemanticIndex {
|
|
36
|
+
private readonly statePath;
|
|
37
|
+
private identity;
|
|
38
|
+
private entries;
|
|
39
|
+
private loaded;
|
|
40
|
+
constructor(options?: {
|
|
41
|
+
readonly statePath?: string;
|
|
42
|
+
});
|
|
43
|
+
/**
|
|
44
|
+
* Embed new/changed chunks, prune removed ones, and persist. Never throws;
|
|
45
|
+
* an embedder failure leaves prior entries intact and reports ok:false.
|
|
46
|
+
*/
|
|
47
|
+
sync(chunks: readonly StoredChunk[], embedder: GistEmbedder): Promise<GistSemanticSyncOk | GistSemanticSyncFail>;
|
|
48
|
+
/** Cosine-ranked chunk ids for a query vector. Never throws. */
|
|
49
|
+
searchByVector(queryVector: readonly number[], topK: number): readonly {
|
|
50
|
+
chunkId: string;
|
|
51
|
+
score: number;
|
|
52
|
+
}[];
|
|
53
|
+
private ensureLoaded;
|
|
54
|
+
private persist;
|
|
55
|
+
}
|
|
56
|
+
export interface GitHubGistSemanticMethodOptions {
|
|
57
|
+
readonly skillName: string;
|
|
58
|
+
readonly skillType: string;
|
|
59
|
+
readonly instanceId: string;
|
|
60
|
+
readonly tags: readonly string[];
|
|
61
|
+
readonly store: DatasourceChunkStore;
|
|
62
|
+
readonly index: GistSemanticIndex;
|
|
63
|
+
readonly embedder: GistEmbedder;
|
|
64
|
+
}
|
|
65
|
+
/** Cosine similarity retrieval over the gist vector sidecar. Never throws. */
|
|
66
|
+
export declare class GitHubGistSemanticMethod implements RetrievalMethod {
|
|
67
|
+
private readonly options;
|
|
68
|
+
constructor(options: GitHubGistSemanticMethodOptions);
|
|
69
|
+
describe(): RetrievalMethodDescriptor;
|
|
70
|
+
retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
|
|
71
|
+
private matchesScope;
|
|
72
|
+
}
|
|
73
|
+
export interface GatewayGistEmbedderOptions {
|
|
74
|
+
/** Runtime resolver; defaults to the shared embedding-runtime ensureRuntime. */
|
|
75
|
+
readonly runtime?: {
|
|
76
|
+
ensureRuntime(input?: {
|
|
77
|
+
profileId?: ProfileId;
|
|
78
|
+
cachedOnly?: boolean;
|
|
79
|
+
}): Promise<{
|
|
80
|
+
baseUrl: string;
|
|
81
|
+
identity: {
|
|
82
|
+
provider: string;
|
|
83
|
+
model: string;
|
|
84
|
+
dimension: number;
|
|
85
|
+
};
|
|
86
|
+
}>;
|
|
87
|
+
};
|
|
88
|
+
readonly profileId?: ProfileId;
|
|
89
|
+
readonly fetchImpl?: typeof fetch;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Production embedder backed by the loopback autorag-gateway. The runtime is
|
|
93
|
+
* ensured lazily with `cachedOnly` so semantic retrieval never triggers a
|
|
94
|
+
* model download; an unprimed cache surfaces as a sync/retrieve degrade.
|
|
95
|
+
*/
|
|
96
|
+
/**
|
|
97
|
+
* Texts per `/v1/embeddings` call. The llama.cpp upstream rejects very large
|
|
98
|
+
* batches (HTTP 413 on a full-archive sync), so embedding fans out in
|
|
99
|
+
* bounded sequential batches while preserving input order.
|
|
100
|
+
*/
|
|
101
|
+
export declare const GIST_EMBED_BATCH_SIZE = 32;
|
|
102
|
+
export declare function createGatewayGistEmbedder(options?: GatewayGistEmbedderOptions): GistEmbedder;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GitHub Gist datasource skill (issue #1588).
|
|
3
|
+
*
|
|
4
|
+
* Composition: {@link GitHubGistConnector} incrementally mirrors the
|
|
5
|
+
* authenticated account's gists into the shared chunk store (lexical BM25),
|
|
6
|
+
* and a {@link GistSemanticIndex} vector sidecar adds semantic retrieval
|
|
7
|
+
* through the loopback embedding gateway. Embedding failures degrade to a
|
|
8
|
+
* `semantic-unavailable` warning — indexing and lexical retrieval continue.
|
|
9
|
+
*/
|
|
10
|
+
import type { RetrievalMethod } from "../../../retrieval/types.js";
|
|
11
|
+
import { ConnectorDatasourceSkill, type ConnectorSkillDefinition, type ConnectorSkillOptions } from "../../connector-skill.js";
|
|
12
|
+
import type { DatasourceIndexResult } from "../../types.js";
|
|
13
|
+
import { GitHubGistConnector, type GitHubGistConnectorOptions } from "./connector.js";
|
|
14
|
+
import { type GatewayGistEmbedderOptions, type GistEmbedder } from "./semantic.js";
|
|
15
|
+
export declare const GITHUB_GIST_SKILL_DEFINITION: ConnectorSkillDefinition;
|
|
16
|
+
export declare function gistDatasourceDir(workspaceRoot: string, skillName: string, instanceId: string): string;
|
|
17
|
+
export interface GitHubGistSemanticOptions {
|
|
18
|
+
/** Set false to run lexical-only. Default true. */
|
|
19
|
+
readonly enabled?: boolean;
|
|
20
|
+
/** Injected embedder (tests); defaults to the loopback gateway embedder. */
|
|
21
|
+
readonly embedder?: GistEmbedder;
|
|
22
|
+
readonly embedderOptions?: GatewayGistEmbedderOptions;
|
|
23
|
+
/** Vector sidecar path override; defaults under `.autorag/datasources/…`. */
|
|
24
|
+
readonly statePath?: string;
|
|
25
|
+
}
|
|
26
|
+
export interface GitHubGistSkillOptions extends Omit<ConnectorSkillOptions, "connector"> {
|
|
27
|
+
/** Trusted connector configuration; a pre-built connector wins. */
|
|
28
|
+
readonly connector?: GitHubGistConnector;
|
|
29
|
+
readonly connectorOptions?: GitHubGistConnectorOptions;
|
|
30
|
+
readonly semantic?: GitHubGistSemanticOptions;
|
|
31
|
+
}
|
|
32
|
+
export declare class GitHubGistSkill extends ConnectorDatasourceSkill {
|
|
33
|
+
private readonly semanticIndex;
|
|
34
|
+
private readonly semanticEmbedder;
|
|
35
|
+
private readonly semanticMethod;
|
|
36
|
+
constructor(options?: GitHubGistSkillOptions);
|
|
37
|
+
index(): Promise<DatasourceIndexResult>;
|
|
38
|
+
retrievalMethods(): readonly RetrievalMethod[];
|
|
39
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { CrawlerCliClient } from "../../crawler-client.js";
|
|
2
|
+
import type { CrawlerSearchResult } from "../../crawler-types.js";
|
|
3
|
+
export interface LarkClientOptions {
|
|
4
|
+
readonly binaryPath?: string;
|
|
5
|
+
readonly env?: Readonly<Record<string, string | undefined>>;
|
|
6
|
+
readonly timeoutMs?: number;
|
|
7
|
+
readonly chatIds?: readonly string[];
|
|
8
|
+
readonly sleep?: (ms: number) => Promise<void>;
|
|
9
|
+
}
|
|
10
|
+
export declare class LarkClient {
|
|
11
|
+
private readonly version;
|
|
12
|
+
private readonly auth;
|
|
13
|
+
private readonly messages;
|
|
14
|
+
private readonly docs;
|
|
15
|
+
private readonly sleep;
|
|
16
|
+
constructor(options?: LarkClientOptions);
|
|
17
|
+
probeVersion(): Promise<CrawlerSearchResult | Awaited<ReturnType<CrawlerCliClient["sync"]>>>;
|
|
18
|
+
probeAuth(): Promise<Awaited<ReturnType<CrawlerCliClient["sync"]>>>;
|
|
19
|
+
searchMessages(query: string, topK: number | undefined): Promise<CrawlerSearchResult>;
|
|
20
|
+
searchDocs(query: string, topK: number | undefined): Promise<CrawlerSearchResult>;
|
|
21
|
+
private retry;
|
|
22
|
+
}
|
|
23
|
+
export declare function isSafeId(value: string): boolean;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { RetrievalMethod, RetrievalOptions, RetrievalResult } from "../../../retrieval/types.js";
|
|
2
|
+
import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
|
|
3
|
+
import { LarkClient, type LarkClientOptions } from "./client.js";
|
|
4
|
+
export interface LarkSkillOptions extends LarkClientOptions {
|
|
5
|
+
readonly client?: LarkClient;
|
|
6
|
+
readonly datasourceId?: string;
|
|
7
|
+
readonly instanceId?: string;
|
|
8
|
+
readonly instances?: readonly string[];
|
|
9
|
+
readonly tags?: readonly string[];
|
|
10
|
+
readonly embedderBaseUrl?: string;
|
|
11
|
+
}
|
|
12
|
+
type Surface = "messages" | "docs";
|
|
13
|
+
export declare class LarkSkill implements DatasourceSkill {
|
|
14
|
+
private readonly client;
|
|
15
|
+
private readonly datasourceId;
|
|
16
|
+
private readonly instanceId;
|
|
17
|
+
private readonly instances;
|
|
18
|
+
private readonly tags;
|
|
19
|
+
private readonly chatIds;
|
|
20
|
+
private readonly embedderBaseUrl;
|
|
21
|
+
private lastIndexedAt;
|
|
22
|
+
private lastError;
|
|
23
|
+
constructor(options?: LarkSkillOptions);
|
|
24
|
+
describe(): DatasourceSkillDescriptor;
|
|
25
|
+
polling(): PollingMetadata;
|
|
26
|
+
index(): Promise<DatasourceIndexResult>;
|
|
27
|
+
retrievalMethods(): readonly RetrievalMethod[];
|
|
28
|
+
describeSources(): readonly SourceDescription[];
|
|
29
|
+
skillManifest(): DatasourceSkillManifest;
|
|
30
|
+
search(surface: Surface, query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
|
|
31
|
+
methodTags(surface: Surface): readonly string[];
|
|
32
|
+
private ok;
|
|
33
|
+
private fail;
|
|
34
|
+
private note;
|
|
35
|
+
}
|
|
36
|
+
export {};
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { LazykatokChunkResult, LazykatokContextResult, LazykatokDoctorResult, LazykatokIndexResult, LazykatokOptions, LazykatokParentResult, LazykatokSearchMode, LazykatokSearchOptions, LazykatokSearchResult, LazykatokSyncResult } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Thin external `lazykatok` CLI wrapper. Every method spawns the `lazykatok` binary as
|
|
4
|
+
* a child process, parses JSON from stdout, and returns a discriminated ok/fail
|
|
5
|
+
* union. No method throws for expected failures (missing binary, CLI nonzero
|
|
6
|
+
* exit, timeout, oversized output, or invalid JSON). The client never opens the
|
|
7
|
+
* KakaoTalk database directly.
|
|
8
|
+
*/
|
|
9
|
+
export declare class LazykatokClient {
|
|
10
|
+
private readonly options;
|
|
11
|
+
constructor(options?: LazykatokOptions);
|
|
12
|
+
doctor(signal?: AbortSignal): Promise<LazykatokDoctorResult>;
|
|
13
|
+
sync(signal?: AbortSignal): Promise<LazykatokSyncResult>;
|
|
14
|
+
index(signal?: AbortSignal): Promise<LazykatokIndexResult>;
|
|
15
|
+
search(mode: LazykatokSearchMode, query: string, options?: LazykatokSearchOptions): Promise<LazykatokSearchResult>;
|
|
16
|
+
chunkGet(chunkId: string, signal?: AbortSignal): Promise<LazykatokChunkResult>;
|
|
17
|
+
context(chunkId: string, signal?: AbortSignal): Promise<LazykatokContextResult>;
|
|
18
|
+
parent(chunkId: string, signal?: AbortSignal): Promise<LazykatokParentResult>;
|
|
19
|
+
/** Single retrieval pipeline: build env, spawn, parse-free raw result. */
|
|
20
|
+
private run;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* `sync` argv for the configured source adapter. A bare `sync --json` makes the CLI
|
|
24
|
+
* fall back to its config file, whose default adapter is `fixture` and fails with
|
|
25
|
+
* "fixture source requires a JSONL path", so the live macOS adapter is named
|
|
26
|
+
* explicitly on darwin while other platforms defer to the CLI's own config.
|
|
27
|
+
*/
|
|
28
|
+
export declare function syncArgs(options: LazykatokOptions, platform?: NodeJS.Platform): readonly string[];
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KakaoTalk datasource skill barrel.
|
|
3
|
+
*
|
|
4
|
+
* Re-exports the retrieval methods and skill plus the underlying client/types
|
|
5
|
+
* from sibling modules. Public source paths are path-opaque virtual paths of
|
|
6
|
+
* the form `/kakao/<instance-id>/chunks/<chunk-id>`.
|
|
7
|
+
*/
|
|
8
|
+
export { LazykatokClient } from "./client.js";
|
|
9
|
+
export { LazykatokBm25Method, type LazykatokMethodOptions, type LazykatokSearchClient, LazykatokSemanticMethod, } from "./methods.js";
|
|
10
|
+
export { LazykatokSkill, type LazykatokSkillClient, type LazykatokSkillOptions } from "./skill.js";
|
|
11
|
+
export type { LazykatokChunkResult, LazykatokDoctorResult, LazykatokFailureReason, LazykatokHit, LazykatokIndexResult, LazykatokOptions, LazykatokSearchMode, LazykatokSearchOptions, LazykatokSearchResult, LazykatokSyncResult, } from "./types.js";
|
|
12
|
+
export { DEFAULT_LAZYKATOK_OPTIONS } from "./types.js";
|
|
@@ -1,42 +1,42 @@
|
|
|
1
1
|
import type { RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult } from "../../../retrieval/types.js";
|
|
2
|
-
import type {
|
|
2
|
+
import type { LazykatokSearchMode, LazykatokSearchOptions, LazykatokSearchResult } from "./types.js";
|
|
3
3
|
/**
|
|
4
4
|
* Narrow client surface required by the KakaoTalk retrieval methods.
|
|
5
|
-
* The real {@link
|
|
5
|
+
* The real {@link LazykatokClient} satisfies this structurally; tests may stub it.
|
|
6
6
|
*/
|
|
7
|
-
export interface
|
|
8
|
-
search(mode:
|
|
7
|
+
export interface LazykatokSearchClient {
|
|
8
|
+
search(mode: LazykatokSearchMode, query: string, options?: LazykatokSearchOptions): Promise<LazykatokSearchResult>;
|
|
9
9
|
}
|
|
10
|
-
export interface
|
|
11
|
-
readonly client:
|
|
10
|
+
export interface LazykatokMethodOptions {
|
|
11
|
+
readonly client: LazykatokSearchClient;
|
|
12
12
|
readonly instanceId: string;
|
|
13
13
|
readonly tags?: readonly string[];
|
|
14
14
|
}
|
|
15
15
|
/**
|
|
16
16
|
* Lexical (BM25) retrieval over a KakaoTalk datasource via the external
|
|
17
|
-
* `
|
|
17
|
+
* `lazykatok` CLI in keyword mode. Lazykatok does not expose source scopes; access is
|
|
18
18
|
* controlled at the datasource/tag level. All client failures collapse to an
|
|
19
19
|
* empty result set; retrieval never throws.
|
|
20
20
|
*/
|
|
21
|
-
export declare class
|
|
21
|
+
export declare class LazykatokBm25Method implements RetrievalMethod {
|
|
22
22
|
private readonly client;
|
|
23
23
|
private readonly instanceId;
|
|
24
24
|
private readonly tags;
|
|
25
|
-
constructor(options:
|
|
25
|
+
constructor(options: LazykatokMethodOptions);
|
|
26
26
|
describe(): RetrievalMethodDescriptor;
|
|
27
27
|
retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
|
|
28
28
|
}
|
|
29
29
|
/**
|
|
30
30
|
* Semantic (vector) retrieval over a KakaoTalk datasource via the external
|
|
31
|
-
* `
|
|
31
|
+
* `lazykatok` CLI in semantic mode. Lazykatok does not expose source scopes; access is
|
|
32
32
|
* controlled at the datasource/tag level. All client failures collapse to an
|
|
33
33
|
* empty result set; retrieval never throws.
|
|
34
34
|
*/
|
|
35
|
-
export declare class
|
|
35
|
+
export declare class LazykatokSemanticMethod implements RetrievalMethod {
|
|
36
36
|
private readonly client;
|
|
37
37
|
private readonly instanceId;
|
|
38
38
|
private readonly tags;
|
|
39
|
-
constructor(options:
|
|
39
|
+
constructor(options: LazykatokMethodOptions);
|
|
40
40
|
describe(): RetrievalMethodDescriptor;
|
|
41
41
|
retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
|
|
42
42
|
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Path helpers for the KakaoTalk (`lazykatok`) datasource skill.
|
|
3
|
+
*
|
|
4
|
+
* All paths are workspace-relative and live under
|
|
5
|
+
* `<workspaceRoot>/.autorag/datasources/lazykatok`. The client never exposes raw
|
|
6
|
+
* filesystem paths to callers; chunk identifiers are surfaced as opaque
|
|
7
|
+
* slash-hierarchical sources of the form
|
|
8
|
+
* `/kakao/<instance-id>/chunks/<chunk-id>`.
|
|
9
|
+
*/
|
|
10
|
+
export declare const AUTORAG_DIRNAME = ".autorag";
|
|
11
|
+
export declare const DATASOURCES_DIRNAME = "datasources";
|
|
12
|
+
export declare const LAZYKATOK_DATASOURCE_DIRNAME = "lazykatok";
|
|
13
|
+
/** Logical datasource kind for source identifiers. */
|
|
14
|
+
export declare const LAZYKATOK_SOURCE_KIND: "kakao";
|
|
15
|
+
/**
|
|
16
|
+
* Root directory for lazykatok-managed state under a workspace. This is where the
|
|
17
|
+
* external `lazykatok` CLI stores its index, chunks, and sync state.
|
|
18
|
+
*/
|
|
19
|
+
export declare function lazykatokDatasourceRoot(workspaceRoot: string): string;
|
|
20
|
+
/** Directory holding the lazykatok chunk index. */
|
|
21
|
+
export declare function lazykatokIndexPath(workspaceRoot: string): string;
|
|
22
|
+
/** Directory holding materialized lazykatok chunks. */
|
|
23
|
+
export declare function lazykatokChunksPath(workspaceRoot: string): string;
|
|
24
|
+
/** Directory holding lazykatok sync state. */
|
|
25
|
+
export declare function lazykatokSyncPath(workspaceRoot: string): string;
|
|
26
|
+
/**
|
|
27
|
+
* Builds the path-opaque, slash-hierarchical source identifier for a single
|
|
28
|
+
* lazykatok chunk. Callers (the LazykatokSkill methods) use this to populate
|
|
29
|
+
* `RetrievalResult.source` so no real filesystem path ever leaks.
|
|
30
|
+
*/
|
|
31
|
+
export declare function lazykatokSourcePath(instanceId: string, chunkId: string): string;
|
|
32
|
+
/**
|
|
33
|
+
* Parses a lazykatok source identifier back into its `{ instanceId, chunkId }`
|
|
34
|
+
* components. Returns `undefined` for malformed or non-kakao sources so that
|
|
35
|
+
* deny decisions remain explicit rather than undefined-as-deny.
|
|
36
|
+
*/
|
|
37
|
+
export declare function parseLazykatokSourcePath(source: string): {
|
|
38
|
+
readonly instanceId: string;
|
|
39
|
+
readonly chunkId: string;
|
|
40
|
+
} | undefined;
|
|
@@ -1,28 +1,28 @@
|
|
|
1
1
|
import type { RetrievalMethod } from "../../../retrieval/types.js";
|
|
2
2
|
import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
|
|
3
|
-
import { type
|
|
4
|
-
import type {
|
|
5
|
-
export interface
|
|
6
|
-
doctor(): Promise<
|
|
7
|
-
sync(): Promise<
|
|
8
|
-
index(): Promise<
|
|
3
|
+
import { type LazykatokSearchClient } from "./methods.js";
|
|
4
|
+
import type { LazykatokDoctorResult, LazykatokIndexResult, LazykatokSyncResult } from "./types.js";
|
|
5
|
+
export interface LazykatokSkillClient extends LazykatokSearchClient {
|
|
6
|
+
doctor(): Promise<LazykatokDoctorResult>;
|
|
7
|
+
sync(): Promise<LazykatokSyncResult>;
|
|
8
|
+
index(): Promise<LazykatokIndexResult>;
|
|
9
9
|
}
|
|
10
|
-
export interface
|
|
11
|
-
readonly client:
|
|
10
|
+
export interface LazykatokSkillOptions {
|
|
11
|
+
readonly client: LazykatokSkillClient;
|
|
12
12
|
readonly instanceId?: string;
|
|
13
13
|
readonly instances?: readonly string[];
|
|
14
14
|
readonly pollingIntervalMs?: number;
|
|
15
15
|
readonly tags?: readonly string[];
|
|
16
16
|
readonly lastIndexedAt?: number;
|
|
17
17
|
}
|
|
18
|
-
export declare class
|
|
18
|
+
export declare class LazykatokSkill implements DatasourceSkill {
|
|
19
19
|
private readonly client;
|
|
20
20
|
private readonly instanceId;
|
|
21
21
|
private readonly instances;
|
|
22
22
|
private readonly pollingIntervalMs;
|
|
23
23
|
private readonly tags;
|
|
24
24
|
private lastIndexedAt;
|
|
25
|
-
constructor(options:
|
|
25
|
+
constructor(options: LazykatokSkillOptions);
|
|
26
26
|
describe(): DatasourceSkillDescriptor;
|
|
27
27
|
polling(): PollingMetadata;
|
|
28
28
|
index(): Promise<DatasourceIndexResult>;
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import type { RetrievalOptions } from "../../../retrieval/types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Search modes accepted by the external `lazykatok search` subcommand.
|
|
4
|
+
*
|
|
5
|
+
* - `semantic` — dense vector similarity over indexed chunks.
|
|
6
|
+
* - `keyword` — exact / tokenized keyword matching.
|
|
7
|
+
* - `hybrid` — fused semantic + keyword scores.
|
|
8
|
+
*/
|
|
9
|
+
export type LazykatokSearchMode = "semantic" | "keyword" | "hybrid";
|
|
10
|
+
/**
|
|
11
|
+
* Configuration for {@link LazykatokClient}. All fields optional; sensible defaults
|
|
12
|
+
* mirror the jikji client. The client spawns the `lazykatok` binary as a child
|
|
13
|
+
* process — it never opens the KakaoTalk database directly.
|
|
14
|
+
*/
|
|
15
|
+
export interface LazykatokOptions {
|
|
16
|
+
/** Explicit path to the `lazykatok` binary. Defaults to a bare `lazykatok` PATH lookup. */
|
|
17
|
+
readonly binaryPath?: string;
|
|
18
|
+
/** Spawn timeout in milliseconds. Default 60_000 (sync/index over large archives can be slow). */
|
|
19
|
+
readonly timeoutMs?: number;
|
|
20
|
+
/** Max stdout/stderr bytes retained. Default 16 MiB (an `index --json` report alone is multiple MB on a real archive). */
|
|
21
|
+
readonly maxBufferBytes?: number;
|
|
22
|
+
/** Explicit native lazykatok data directory, passed as `--data-dir`. */
|
|
23
|
+
readonly workspacePath?: string;
|
|
24
|
+
/** Explicit operator-owned configuration/workspace transport. */
|
|
25
|
+
readonly configPath?: string;
|
|
26
|
+
/**
|
|
27
|
+
* Source adapter passed to `sync --source`. Without it the CLI falls back to its
|
|
28
|
+
* config file, whose default adapter is `fixture` and fails without a JSONL path.
|
|
29
|
+
* AutoRAG names the live adapter (`macos`) on macOS and passes no flag elsewhere,
|
|
30
|
+
* so a non-macOS operator's own CLI config still decides; set this to use another
|
|
31
|
+
* adapter (`kakaocli`, `fixture`, …).
|
|
32
|
+
*/
|
|
33
|
+
readonly source?: string;
|
|
34
|
+
/** Environment overrides merged on top of `process.env` for the child. */
|
|
35
|
+
readonly env?: Readonly<Record<string, string | undefined>>;
|
|
36
|
+
}
|
|
37
|
+
export declare const DEFAULT_LAZYKATOK_BINARY = "lazykatok";
|
|
38
|
+
export declare const DEFAULT_LAZYKATOK_TIMEOUT_MS = 60000;
|
|
39
|
+
/** The live macOS adapter `sync` names by default on darwin. */
|
|
40
|
+
export declare const DEFAULT_LAZYKATOK_SOURCE = "macos";
|
|
41
|
+
export declare const DEFAULT_LAZYKATOK_MAX_BUFFER_BYTES: number;
|
|
42
|
+
export declare const DEFAULT_LAZYKATOK_OPTIONS: {
|
|
43
|
+
readonly binaryPath: "lazykatok";
|
|
44
|
+
readonly timeoutMs: 60000;
|
|
45
|
+
readonly maxBufferBytes: number;
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* Reasons a lazykatok CLI invocation can fail. The client never throws for these —
|
|
49
|
+
* every method returns a discriminated union with `ok: false` and one of these
|
|
50
|
+
* reasons.
|
|
51
|
+
*/
|
|
52
|
+
export type LazykatokFailureReason = "binary-missing" | "nonzero-exit" | "spawn-error" | "timeout" | "aborted" | "stdout-too-large" | "stderr-too-large" | "invalid-json" | "invalid-shape";
|
|
53
|
+
/** Common failure payload shared by every method result union. */
|
|
54
|
+
export interface LazykatokFailure {
|
|
55
|
+
readonly ok: false;
|
|
56
|
+
readonly reason: LazykatokFailureReason;
|
|
57
|
+
/** Raw stdout captured (may be partial / empty). Never a binary path. */
|
|
58
|
+
readonly stdout: string;
|
|
59
|
+
/** Raw stderr captured (may be partial / empty). Never a binary path. */
|
|
60
|
+
readonly stderr: string;
|
|
61
|
+
/** Exit code, or `null` when the process never exited normally. */
|
|
62
|
+
readonly code: number | null;
|
|
63
|
+
readonly hits?: readonly unknown[];
|
|
64
|
+
}
|
|
65
|
+
/** A single indexed chat chunk returned by the lazykatok CLI. */
|
|
66
|
+
export interface LazykatokChunk {
|
|
67
|
+
readonly chunkId: string;
|
|
68
|
+
readonly content: string;
|
|
69
|
+
readonly metadata?: Readonly<Record<string, unknown>>;
|
|
70
|
+
}
|
|
71
|
+
/** A scored search hit. */
|
|
72
|
+
export interface LazykatokSearchHit {
|
|
73
|
+
readonly chunkId: string;
|
|
74
|
+
readonly score: number;
|
|
75
|
+
readonly content: string;
|
|
76
|
+
readonly metadata?: Readonly<Record<string, unknown>>;
|
|
77
|
+
readonly source?: string;
|
|
78
|
+
}
|
|
79
|
+
export type LazykatokHit = LazykatokSearchHit;
|
|
80
|
+
/** Result of `lazykatok doctor`. */
|
|
81
|
+
export interface LazykatokDoctorInfo {
|
|
82
|
+
readonly version?: string;
|
|
83
|
+
readonly ready: boolean;
|
|
84
|
+
readonly metadata?: Readonly<Record<string, unknown>>;
|
|
85
|
+
}
|
|
86
|
+
/** Result of `lazykatok sync`. */
|
|
87
|
+
export interface LazykatokSyncInfo {
|
|
88
|
+
readonly synced: boolean;
|
|
89
|
+
readonly messageCount?: number;
|
|
90
|
+
readonly metadata?: Readonly<Record<string, unknown>>;
|
|
91
|
+
}
|
|
92
|
+
/** Result of `lazykatok index`. */
|
|
93
|
+
export interface LazykatokIndexInfo {
|
|
94
|
+
readonly chunkCount: number;
|
|
95
|
+
readonly metadata?: Readonly<Record<string, unknown>>;
|
|
96
|
+
}
|
|
97
|
+
/** Context payload: the chunks surrounding a target chunk. */
|
|
98
|
+
export interface LazykatokContext {
|
|
99
|
+
readonly chunks: readonly LazykatokChunk[];
|
|
100
|
+
readonly metadata?: Readonly<Record<string, unknown>>;
|
|
101
|
+
}
|
|
102
|
+
/** Discriminated ok-success shape carrying typed `data`. */
|
|
103
|
+
export interface LazykatokOk<T> {
|
|
104
|
+
readonly ok: true;
|
|
105
|
+
readonly data: T;
|
|
106
|
+
readonly stdout: string;
|
|
107
|
+
readonly stderr: string;
|
|
108
|
+
readonly code: number;
|
|
109
|
+
}
|
|
110
|
+
export interface LazykatokSearchOk {
|
|
111
|
+
readonly ok: true;
|
|
112
|
+
readonly hits: readonly LazykatokSearchHit[];
|
|
113
|
+
readonly stdout: string;
|
|
114
|
+
readonly stderr: string;
|
|
115
|
+
readonly code: number;
|
|
116
|
+
readonly data: {
|
|
117
|
+
readonly hits: readonly LazykatokSearchHit[];
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
export type LazykatokDoctorResult = LazykatokOk<LazykatokDoctorInfo> | LazykatokFailure;
|
|
121
|
+
export type LazykatokSyncResult = LazykatokOk<LazykatokSyncInfo> | LazykatokFailure;
|
|
122
|
+
export type LazykatokIndexResult = LazykatokOk<LazykatokIndexInfo> | LazykatokFailure;
|
|
123
|
+
export type LazykatokSearchResult = LazykatokSearchOk | LazykatokFailure;
|
|
124
|
+
export type LazykatokChunkResult = LazykatokOk<LazykatokChunk> | LazykatokFailure;
|
|
125
|
+
export type LazykatokContextResult = LazykatokOk<LazykatokContext> | LazykatokFailure;
|
|
126
|
+
/** The real `chunk parent --json` payload is a list of parent windows. */
|
|
127
|
+
export type LazykatokParentResult = LazykatokOk<readonly LazykatokChunk[]> | LazykatokFailure;
|
|
128
|
+
/**
|
|
129
|
+
* Search-specific options. Reuses the shared {@link RetrievalOptions} so the
|
|
130
|
+
* LazykatokSkill retrieval method can pass its options straight through. The
|
|
131
|
+
* client maps `topK` to the native CLI limit flag; other fields are handled
|
|
132
|
+
* by the retrieval layer or are not supported by lazykatok.
|
|
133
|
+
*/
|
|
134
|
+
export type LazykatokSearchOptions = RetrievalOptions;
|
|
@@ -10,6 +10,8 @@ export declare class MailcrawlClient implements MailcrawlSearchClient {
|
|
|
10
10
|
constructor(options?: MailcrawlOptions);
|
|
11
11
|
sync(signal?: AbortSignal): Promise<MailcrawlSyncResult>;
|
|
12
12
|
search(mode: MailcrawlSearchMode, query: string, options?: MailcrawlSearchOptions): Promise<MailcrawlSearchResult>;
|
|
13
|
+
/** `sync`/`index` share the long non-interactive budget. */
|
|
14
|
+
private indexTimeoutMs;
|
|
13
15
|
private run;
|
|
14
16
|
index(signal?: AbortSignal): Promise<MailcrawlOk<MailcrawlIndexInfo> | MailcrawlFailure>;
|
|
15
17
|
}
|