@autorag/librarian 2.1.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.md +155 -55
  2. package/dist/agent/agent.d.ts +27 -41
  3. package/dist/agent/bash-tool.d.ts +0 -11
  4. package/dist/agent/dupey-tool.d.ts +15 -0
  5. package/dist/agent/emit-results-tool.d.ts +14 -0
  6. package/dist/agent/index.d.ts +1 -0
  7. package/dist/agent/local-model.d.ts +12 -0
  8. package/dist/agent/search-bm25-tool.d.ts +1 -1
  9. package/dist/agent/search-documents.d.ts +1 -1
  10. package/dist/agent/search-minsync-tool.d.ts +1 -1
  11. package/dist/agent/system-prompt.d.ts +1 -2
  12. package/dist/cli/commands/duplicates.d.ts +4 -0
  13. package/dist/cli/commands/health.d.ts +9 -65
  14. package/dist/cli/commands/init.d.ts +1 -1
  15. package/dist/cli/commands/search.d.ts +2 -2
  16. package/dist/cli/config.d.ts +10 -14
  17. package/dist/cli/index.js +10652 -9947
  18. package/dist/datasource/aliased-skill.d.ts +28 -0
  19. package/dist/datasource/chunk-store.d.ts +2 -0
  20. package/dist/datasource/connector-skill.d.ts +4 -1
  21. package/dist/datasource/connector.d.ts +5 -1
  22. package/dist/datasource/crawler-client.d.ts +9 -0
  23. package/dist/datasource/crawler-skill.d.ts +47 -0
  24. package/dist/datasource/crawler-types.d.ts +53 -0
  25. package/dist/datasource/index.d.ts +8 -4
  26. package/dist/datasource/skills/cloud-drive/index.d.ts +1 -0
  27. package/dist/datasource/skills/cloud-drive/skill.d.ts +14 -0
  28. package/dist/datasource/skills/discrawl/client.d.ts +23 -0
  29. package/dist/datasource/skills/discrawl/config.d.ts +9 -0
  30. package/dist/datasource/skills/discrawl/index.d.ts +5 -0
  31. package/dist/datasource/skills/discrawl/methods.d.ts +55 -0
  32. package/dist/datasource/skills/discrawl/paths.d.ts +30 -0
  33. package/dist/datasource/skills/discrawl/skill.d.ts +61 -0
  34. package/dist/datasource/skills/discrawl/types.d.ts +198 -0
  35. package/dist/datasource/skills/factory.d.ts +7 -0
  36. package/dist/datasource/skills/gdrive/rclone-connector.d.ts +16 -24
  37. package/dist/datasource/skills/notion/index.d.ts +15 -2
  38. package/dist/datasource/skills/obsidian/client.d.ts +15 -0
  39. package/dist/datasource/skills/obsidian/index.d.ts +6 -2
  40. package/dist/datasource/skills/obsidian/methods.d.ts +26 -0
  41. package/dist/datasource/skills/obsidian/paths.d.ts +27 -0
  42. package/dist/datasource/skills/obsidian/skill.d.ts +41 -17
  43. package/dist/datasource/skills/obsidian/types.d.ts +90 -0
  44. package/dist/datasource/skills/slack/index.d.ts +15 -2
  45. package/dist/datasource/skills/telecrawl/index.d.ts +15 -0
  46. package/dist/datasource/skills/wacrawl/index.d.ts +15 -0
  47. package/dist/dupey/cli.d.ts +40 -0
  48. package/dist/dupey/filter.d.ts +8 -0
  49. package/dist/dupey/index.d.ts +2 -0
  50. package/dist/index.d.ts +1 -0
  51. package/dist/index.js +7774 -7409
  52. package/dist/memory/check-memory-tool.d.ts +1 -0
  53. package/dist/memory/index.d.ts +2 -2
  54. package/dist/memory/memory.d.ts +33 -3
  55. package/dist/memory/renderer.d.ts +2 -1
  56. package/dist/mirror/sync.d.ts +3 -1
  57. package/dist/observability/run-log.d.ts +1 -26
  58. package/dist/process/portable-spawn.d.ts +10 -0
  59. package/dist/retrieval/merger.d.ts +1 -0
  60. package/dist/retrieval/scope.d.ts +10 -0
  61. package/package.json +21 -20
  62. package/skills/autorag/SKILL.md +59 -96
  63. package/skills/autorag-setup/SKILL.md +144 -378
  64. package/dist/datasource/skills/discord/connector.d.ts +0 -30
  65. package/dist/datasource/skills/discord/index.d.ts +0 -2
  66. package/dist/datasource/skills/discord/skill.d.ts +0 -20
  67. package/dist/datasource/skills/notion/connector.d.ts +0 -26
  68. package/dist/datasource/skills/notion/skill.d.ts +0 -20
  69. package/dist/datasource/skills/obsidian/connector.d.ts +0 -22
  70. package/dist/datasource/skills/slack/connector.d.ts +0 -28
  71. package/dist/datasource/skills/slack/skill.d.ts +0 -20
  72. package/dist/subagents/contracts.d.ts +0 -90
  73. package/dist/subagents/dispatch-templates.d.ts +0 -104
  74. package/dist/subagents/dispatch-validation.d.ts +0 -169
  75. package/dist/subagents/explorer-tools-extension.d.ts +0 -4
  76. package/dist/subagents/explorer-tools-extension.js +0 -130
  77. package/dist/subagents/local-models.d.ts +0 -14
  78. package/dist/subagents/model-policy.d.ts +0 -31
  79. package/dist/subagents/runtime.d.ts +0 -52
@@ -14,5 +14,5 @@ export interface SearchBM25DocumentsDetails {
14
14
  readonly readiness: BM25Status["readiness"];
15
15
  readonly engine: BM25Status["engine"];
16
16
  }
17
- export declare function createSearchBM25DocumentsTool(getMethod: () => BM25Method | undefined): AgentTool<typeof searchBM25Schema, SearchBM25DocumentsDetails>;
17
+ export declare function createSearchBM25DocumentsTool(getMethod: () => BM25Method | undefined, resolveScope?: (scope: string | undefined) => string | undefined): AgentTool<typeof searchBM25Schema, SearchBM25DocumentsDetails>;
18
18
  export {};
@@ -9,7 +9,7 @@ export type SearchDocumentDiagnosticSeverity = "info" | "warning" | "error";
9
9
  * (#21/#22) does not require another source-breaking type change; codes not yet
10
10
  * emitted are still valid members.
11
11
  */
12
- export type SearchDocumentDiagnosticCode = "empty-query" | "unknown-warning" | "caller-tool-dropped" | "bm25-unavailable" | "bm25-degraded-fallback" | "minsync-unavailable" | "parser-skipped" | "parser-failed" | "unsupported-file" | "stale-index" | "deleted-mirror" | "retrieval-method-failed" | "jikji-unavailable" | "jikji-prepare-failed" | "jikji-find-failed" | "refresh-failed" | "watch-failed" | "watch-limited";
12
+ export type SearchDocumentDiagnosticCode = "empty-query" | "unknown-warning" | "caller-tool-dropped" | "bm25-unavailable" | "bm25-degraded-fallback" | "minsync-unavailable" | "parser-skipped" | "parser-failed" | "duplicate-excluded" | "unsupported-file" | "stale-index" | "deleted-mirror" | "retrieval-method-failed" | "jikji-unavailable" | "jikji-prepare-failed" | "jikji-find-failed" | "refresh-failed" | "watch-failed" | "watch-limited";
13
13
  export interface SearchDocumentDiagnostic {
14
14
  readonly code: SearchDocumentDiagnosticCode;
15
15
  readonly severity: SearchDocumentDiagnosticSeverity;
@@ -20,5 +20,5 @@ export interface SearchMinSyncDocumentsDetails {
20
20
  * tool returns a path-free unavailable message with `available: false` and a
21
21
  * zero result count — never a real binary or workspace path.
22
22
  */
23
- export declare function createSearchMinSyncDocumentsTool(getMethod: () => MinSyncVectorMethod | undefined): AgentTool<typeof searchMinSyncSchema, SearchMinSyncDocumentsDetails>;
23
+ export declare function createSearchMinSyncDocumentsTool(getMethod: () => MinSyncVectorMethod | undefined, resolveScope?: (scope: string | undefined) => string | undefined): AgentTool<typeof searchMinSyncSchema, SearchMinSyncDocumentsDetails>;
24
24
  export {};
@@ -2,8 +2,7 @@ import type { Skill } from "@earendil-works/pi-agent-core";
2
2
  import type { StoreManifest } from "../manifest/types.js";
3
3
  export interface SystemPromptConfig {
4
4
  toolNames: string[];
5
- orchestratorModelId?: string;
6
- explorerModelId?: string;
5
+ modelId?: string;
7
6
  memorySignalCount?: number;
8
7
  memoryEntries?: readonly unknown[];
9
8
  manifests: StoreManifest[];
@@ -0,0 +1,4 @@
1
+ import type { DupeyScanResult } from "../../dupey/index.js";
2
+ import type { CommandContext } from "./types.js";
3
+ export type DuplicateScanner = (path: string) => Promise<DupeyScanResult>;
4
+ export declare function runDuplicates(ctx: CommandContext, scanner?: DuplicateScanner): Promise<number>;
@@ -1,12 +1,10 @@
1
1
  import type { Api, Model } from "@earendil-works/pi-ai";
2
- import type { LoadLocalAutoRAGModelsOptions } from "../../subagents/local-models.js";
3
- import { type CliConfig } from "../config.js";
2
+ import type { LoadLocalAutoRAGModelOptions } from "../../agent/local-model.js";
3
+ import { type CliConfig, type ResolvedAgentModelDetailed } from "../config.js";
4
4
  import type { CommandContext } from "./types.js";
5
- export type HealthCategory = "config" | "model_resolution" | "auth_missing" | "provider_unreachable" | "completion_failed" | "subagent_failed" | "timeout" | "ok";
6
- export type HealthRole = "orchestrator" | "explorer";
5
+ export type HealthCategory = "config" | "model_resolution" | "auth_missing" | "provider_unreachable" | "completion_failed" | "timeout" | "ok";
7
6
  export interface HealthCoverage {
8
7
  modelProvider: boolean;
9
- subagentDispatch: boolean;
10
8
  retrievalTools: false;
11
9
  searchCuration: false;
12
10
  indexHealth: false;
@@ -17,12 +15,10 @@ export interface HealthRoleAuth {
17
15
  envName?: string;
18
16
  }
19
17
  export interface HealthRoleReport {
20
- role: HealthRole;
21
18
  provider: string;
22
19
  modelId: string;
23
20
  displayName?: string;
24
21
  api: string;
25
- routeFamily?: string;
26
22
  baseUrl?: string;
27
23
  contextWindow?: number;
28
24
  maxTokens?: number;
@@ -35,7 +31,6 @@ export interface HealthRoleReport {
35
31
  resolutionSource: "config" | "flags" | "env" | "local_runtime" | "catalog" | "configured_alias" | "mixed";
36
32
  }
37
33
  export interface HealthProbeReport {
38
- role: HealthRole;
39
34
  skipped: boolean;
40
35
  ok: boolean;
41
36
  category: HealthCategory;
@@ -55,75 +50,32 @@ export interface HealthReportV1 {
55
50
  probesSkipped: boolean;
56
51
  coverage: HealthCoverage;
57
52
  config: HealthConfigReport;
58
- models: {
59
- orchestrator?: HealthRoleReport;
60
- explorer?: HealthRoleReport;
61
- };
62
- probes: {
63
- orchestrator?: HealthProbeReport;
64
- explorer?: HealthProbeReport;
65
- };
53
+ model?: HealthRoleReport;
54
+ probe?: HealthProbeReport;
66
55
  indexHealth: {
67
56
  separate: true;
68
57
  command: "autorag status";
69
58
  included: false;
70
59
  };
71
60
  }
72
- /**
73
- * Probe input passed to injected orchestrator/explorer probes. Credential
74
- * values are present for the probe to use but must never be surfaced in
75
- * rendered output.
76
- */
77
61
  export interface ProbeInput {
78
- role: HealthRole;
79
62
  model: Model<Api>;
80
63
  apiKey?: string;
81
64
  providerApiKeys?: Readonly<Record<string, string>>;
82
65
  timeoutMs: number;
83
66
  cwd: string;
84
67
  }
85
- /**
86
- * Probe outcome returned by injected probes. `category` is the per-role
87
- * health category; `message` is already sanitized (no secrets/paths/stacks).
88
- */
89
68
  export interface ProbeOutput {
90
69
  ok: boolean;
91
70
  category: HealthCategory;
92
71
  message?: string;
93
72
  }
94
- /**
95
- * Detailed role metadata produced by the model resolver. The default
96
- * adapter derives this from {@link resolveAgentModel}; when config exports a
97
- * dedicated `resolveAgentModelDetailed`, the health command prefers it.
98
- */
99
- export interface ResolvedHealthRole {
100
- readonly provider: string;
101
- readonly modelId: string;
102
- readonly displayName?: string;
103
- readonly api: Api;
104
- readonly baseUrl?: string;
105
- readonly contextWindow?: number;
106
- readonly maxTokens?: number;
107
- readonly capabilities: {
108
- input: readonly string[];
109
- reasoning: boolean;
110
- };
111
- readonly auth: {
112
- present: boolean;
113
- source: "env" | "local_runtime" | "pi_auth" | "catalog" | "none" | "unknown";
114
- envName?: string;
115
- };
116
- readonly resolutionSource: "config" | "flags" | "env" | "local_runtime" | "catalog" | "configured_alias" | "mixed";
117
- }
73
+ export type ResolvedHealthRole = ResolvedAgentModelDetailed["role"];
118
74
  export interface ResolvedHealthModel {
119
75
  readonly model: Model<Api>;
120
- readonly explorerModel: Model<Api>;
121
76
  readonly apiKey?: string;
122
77
  readonly providerApiKeys?: Readonly<Record<string, string>>;
123
- readonly roles: {
124
- orchestrator: ResolvedHealthRole;
125
- explorer: ResolvedHealthRole;
126
- };
78
+ readonly role: ResolvedHealthRole;
127
79
  }
128
80
  export interface HealthDeps {
129
81
  configResolver?: (input: {
@@ -131,16 +83,8 @@ export interface HealthDeps {
131
83
  cwd?: string;
132
84
  env?: NodeJS.ProcessEnv;
133
85
  }) => CliConfig;
134
- modelResolver?: (config: CliConfig, localOptions?: LoadLocalAutoRAGModelsOptions) => ResolvedHealthModel;
135
- orchestratorProbe?: (input: ProbeInput, signal: AbortSignal) => Promise<ProbeOutput>;
136
- explorerProbe?: (input: ProbeInput, signal: AbortSignal) => Promise<ProbeOutput>;
86
+ modelResolver?: (config: CliConfig, localOptions?: LoadLocalAutoRAGModelOptions) => ResolvedHealthModel;
87
+ probe?: (input: ProbeInput, signal: AbortSignal) => Promise<ProbeOutput>;
137
88
  now?: () => number;
138
89
  }
139
- /**
140
- * Run the `autorag health` command. Returns an exit code and writes the
141
- * rendered health report to `ctx.stdout` (or an error to `ctx.stderr`).
142
- *
143
- * All network/runtime behavior is injectable via `deps`; unit tests inject
144
- * fake resolvers/probes and never touch live networks.
145
- */
146
90
  export declare function runHealth(ctx: CommandContext, deps?: HealthDeps): Promise<number>;
@@ -1,7 +1,7 @@
1
1
  import type { CommandContext } from "./types.js";
2
2
  /**
3
3
  * `autorag init` writes the default config to `~/.autorag/config.json`.
4
- * Role-specific model flags are folded into the generated config. A legacy
4
+ * Model flags are folded into the generated config. A legacy
5
5
  * `autorag.config.json` in the current working directory may be migrated
6
6
  * non-destructively. An existing home config is not overwritten without
7
7
  * `--force`.
@@ -4,12 +4,12 @@ import type { CommandContext } from "./types.js";
4
4
  /**
5
5
  * A recoverable health hint surfaced when `autorag search` fails for a reason
6
6
  * that `autorag health` can diagnose. The classifier only produces hints for
7
- * model/provider/subagent failures — never for retrieval, index, datasource,
7
+ * model/provider failures — never for retrieval, index, datasource,
8
8
  * or empty-query errors.
9
9
  */
10
10
  export interface SearchHealthHint {
11
11
  command: "autorag health";
12
- reason: "model_resolution" | "subagent_failed" | "auth_missing" | "provider_unreachable" | "timeout";
12
+ reason: "model_resolution" | "auth_missing" | "provider_unreachable" | "timeout";
13
13
  message: string;
14
14
  }
15
15
  /**
@@ -1,10 +1,10 @@
1
1
  import type { Api, Model } from "@earendil-works/pi-ai";
2
2
  import type { AutoRAGAgentOptions } from "../agent/agent.js";
3
+ import { type LoadLocalAutoRAGModelOptions } from "../agent/local-model.js";
3
4
  import type { DatasourceAccessContextOptions } from "../datasource/access-context.js";
4
5
  import { type DatasourcesConfig } from "../datasource/skills/factory.js";
5
6
  import type { EnsureMinSyncBinaryOptions, MinSyncEmbedderConfig } from "../minsync/index.js";
6
7
  import type { BM25Engine, BM25FallbackMode } from "../retrieval/methods/bm25.js";
7
- import { type LoadLocalAutoRAGModelsOptions } from "../subagents/local-models.js";
8
8
  export declare const DEFAULT_CONFIG_FILENAME = "config.json";
9
9
  export declare const LEGACY_CONFIG_FILENAME = "autorag.config.json";
10
10
  export { AUTORAG_HOME_ENV, resolveAutoRAGHome } from "../config/home.js";
@@ -76,16 +76,17 @@ export interface CliConfig {
76
76
  searchPaths: string[];
77
77
  workspacePath: string;
78
78
  memoryPath: string;
79
- /** @deprecated Prefer agents.orchestrator. Kept for legacy single-model configs. */
80
79
  model?: AgentModelConfig;
81
- agents?: {
82
- orchestrator?: AgentModelConfig;
83
- explorer?: AgentModelConfig;
84
- };
85
80
  minSync?: MinSyncMethodConfig;
86
81
  bm25?: Bm25MethodConfig;
87
82
  jikji?: Record<string, unknown>;
88
83
  parserOptions?: Record<string, unknown>;
84
+ dupey?: {
85
+ enabled?: boolean;
86
+ binaryPath?: string;
87
+ timeoutMs?: number;
88
+ };
89
+ excludeExactDuplicates?: boolean;
89
90
  /** Trusted datasource skill configuration (skill name → config). */
90
91
  datasources?: DatasourcesConfig;
91
92
  /** Trusted datasource allow-tags/allow-scopes. Absent ⇒ default-deny. */
@@ -133,7 +134,6 @@ export declare function buildAgentOptions(config: CliConfig): Omit<AutoRAGAgentO
133
134
  export declare function resolveModel(config: CliConfig): Model<Api>;
134
135
  export interface ResolvedAgentModel {
135
136
  readonly model: Model<Api>;
136
- readonly explorerModel: Model<Api>;
137
137
  readonly apiKey?: string;
138
138
  readonly providerApiKeys?: Readonly<Record<string, string>>;
139
139
  }
@@ -160,16 +160,12 @@ export interface ResolvedAgentModelRole {
160
160
  }
161
161
  export interface ResolvedAgentModelDetailed {
162
162
  readonly model: Model<Api>;
163
- readonly explorerModel: Model<Api>;
164
163
  readonly apiKey?: string;
165
164
  readonly providerApiKeys?: Readonly<Record<string, string>>;
166
- readonly roles: {
167
- readonly orchestrator: ResolvedAgentModelRole;
168
- readonly explorer: ResolvedAgentModelRole;
169
- };
165
+ readonly role: ResolvedAgentModelRole;
170
166
  }
171
- export declare function resolveAgentModel(config: CliConfig, localOptions?: LoadLocalAutoRAGModelsOptions): ResolvedAgentModel;
172
- export declare function resolveAgentModelDetailed(config: CliConfig, localOptions?: LoadLocalAutoRAGModelsOptions): ResolvedAgentModelDetailed;
167
+ export declare function resolveAgentModel(config: CliConfig, localOptions?: LoadLocalAutoRAGModelOptions): ResolvedAgentModel;
168
+ export declare function resolveAgentModelDetailed(config: CliConfig, localOptions?: LoadLocalAutoRAGModelOptions): ResolvedAgentModelDetailed;
173
169
  export declare function writeDefaultConfig(path: string, partial: Partial<CliConfig>, opts?: {
174
170
  force?: boolean;
175
171
  atomicCreate?: boolean;