greptor 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -6
- package/dist/greptor.d.ts.map +1 -1
- package/dist/greptor.js +3 -1
- package/dist/greptor.js.map +1 -1
- package/dist/llm/llm-factory.d.ts +7 -0
- package/dist/llm/llm-factory.d.ts.map +1 -0
- package/dist/llm/llm-factory.js +52 -0
- package/dist/llm/llm-factory.js.map +1 -0
- package/dist/processing/processor.d.ts.map +1 -1
- package/dist/processing/processor.js +48 -18
- package/dist/processing/processor.js.map +1 -1
- package/dist/types.d.ts +21 -2
- package/dist/types.d.ts.map +1 -1
- package/package.json +19 -1
package/README.md
CHANGED
|
@@ -34,23 +34,59 @@ bun add greptor
|
|
|
34
34
|
|
|
35
35
|
### Step 2: Initialize
|
|
36
36
|
|
|
37
|
-
Create a Greptor instance with your base directory, topic, and
|
|
37
|
+
Create a Greptor instance with your base directory, topic, and model config.
|
|
38
38
|
|
|
39
39
|
```typescript
|
|
40
40
|
import { createGreptor } from 'greptor';
|
|
41
|
-
import { openai } from "@ai-sdk/openai";
|
|
42
41
|
|
|
43
42
|
// Create Greptor instance
|
|
44
43
|
const greptor = await createGreptor({
|
|
45
44
|
baseDir: './projects/investing',
|
|
46
45
|
topic: 'Investing, stock market, financial, and macroeconomics',
|
|
47
|
-
model:
|
|
46
|
+
model: {
|
|
47
|
+
provider: "@ai-sdk/openai",
|
|
48
|
+
model: "gpt-5-mini",
|
|
49
|
+
},
|
|
48
50
|
});
|
|
49
51
|
```
|
|
50
52
|
|
|
51
53
|
- **`baseDir`**: Home directory for your project where all data will be stored.
|
|
52
54
|
- **`topic`**: Helps Greptor understand your data better and generate a relevant tag schema.
|
|
53
|
-
- **`model`**: A `
|
|
55
|
+
- **`model`**: A config object with `provider`, `model`, and optional `options` for the [Vercel AI SDK](https://ai-sdk.dev).
|
|
56
|
+
|
|
57
|
+
#### Model Config
|
|
58
|
+
|
|
59
|
+
Greptor uses an LLM (via [Greptor](https://github.com/greptorio/greptor)) to process content. You'll need to:
|
|
60
|
+
|
|
61
|
+
1. **Choose a provider** from the [AI SDK ecosystem](https://sdk.vercel.ai/providers/ai-sdk-providers):
|
|
62
|
+
- `@ai-sdk/openai` - OpenAI (GPT-4, GPT-4o, etc.)
|
|
63
|
+
- `@ai-sdk/anthropic` - Anthropic (Claude)
|
|
64
|
+
- `@ai-sdk/groq` - Groq (fast inference)
|
|
65
|
+
- `@ai-sdk/openai-compatible` - OpenAI-compatible endpoints (NVIDIA NIM, OpenRouter, etc.)
|
|
66
|
+
- And [many more](https://sdk.vercel.ai/providers/ai-sdk-providers)...
|
|
67
|
+
|
|
68
|
+
2. **Get an API key** from your provider and set it as an environment variable:
|
|
69
|
+
```bash
|
|
70
|
+
export OPENAI_API_KEY="sk-..."
|
|
71
|
+
# or add to ~/.bashrc, ~/.zshrc, etc.
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
3. **Provide it in the model config** when creating Greptor.
|
|
75
|
+
```typescript
|
|
76
|
+
const greptor = await createGreptor({
|
|
77
|
+
baseDir: './projects/investing',
|
|
78
|
+
topic: 'Investing, stock market, financial, and macroeconomics',
|
|
79
|
+
model: {
|
|
80
|
+
provider: "@ai-sdk/openai-compatible",
|
|
81
|
+
model: "z-ai/glm4.7",
|
|
82
|
+
name: "nvidia",
|
|
83
|
+
options: {
|
|
84
|
+
baseURL: "https://integrate.api.nvidia.com/v1",
|
|
85
|
+
apiKey: process.env.NVIDIA_API_KEY,
|
|
86
|
+
},
|
|
87
|
+
},
|
|
88
|
+
});
|
|
89
|
+
```
|
|
54
90
|
|
|
55
91
|
### Step 3: Start Feeding Documents
|
|
56
92
|
|
|
@@ -304,7 +340,10 @@ Greptor provides optional hooks to monitor the ingestion and processing pipeline
|
|
|
304
340
|
const greptor = await createGreptor({
|
|
305
341
|
baseDir: './projects/investing',
|
|
306
342
|
topic: 'Investing, stock market, financial, and macroeconomics',
|
|
307
|
-
model:
|
|
343
|
+
model: {
|
|
344
|
+
provider: "@ai-sdk/openai",
|
|
345
|
+
model: "gpt-5-mini",
|
|
346
|
+
},
|
|
308
347
|
hooks: {
|
|
309
348
|
onProcessingRunStarted: ({ documentsToProcess, totalDocuments }) => {
|
|
310
349
|
console.log(`📋 Starting processing run: ${documentsToProcess} documents queued`);
|
|
@@ -375,7 +414,10 @@ Here's a comprehensive example for investment research:
|
|
|
375
414
|
const greptor = await createGreptor({
|
|
376
415
|
baseDir: './projects/investing',
|
|
377
416
|
topic: 'Investing, stock market, financial, and macroeconomics',
|
|
378
|
-
model:
|
|
417
|
+
model: {
|
|
418
|
+
provider: "@ai-sdk/openai",
|
|
419
|
+
model: "gpt-5-mini",
|
|
420
|
+
},
|
|
379
421
|
tagSchema: [
|
|
380
422
|
{
|
|
381
423
|
name: 'company',
|
package/dist/greptor.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"greptor.d.ts","sourceRoot":"","sources":["../src/greptor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,cAAc,EACd,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"greptor.d.ts","sourceRoot":"","sources":["../src/greptor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,cAAc,EACd,MAAM,YAAY,CAAC;AAcpB,MAAM,WAAW,OAAO;IACvB,GAAG,EAAE,CAAC,KAAK,EAAE,eAAe,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC3D,WAAW,EAAE,CACZ,OAAO,EAAE,MAAM,EAAE,EACjB,SAAS,EAAE,OAAO,KACd,OAAO,CAAC,iBAAiB,CAAC,CAAC;CAChC;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,CAoH7E"}
|
package/dist/greptor.js
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
import YAML from "yaml";
|
|
3
|
+
import { resolveModel } from "./llm/llm-factory.js";
|
|
3
4
|
import { createProcessingQueue, enqueueUnprocessedDocuments, startBackgroundWorkers, } from "./processing/processor.js";
|
|
4
5
|
import { generateSkill } from "./skills/skill-generator.js";
|
|
5
6
|
import { createFileStorage } from "./storage/file-storage.js";
|
|
6
7
|
import { initializeTagSchema } from "./tag-schema/initialize.js";
|
|
7
8
|
export async function createGreptor(options) {
|
|
8
|
-
const { baseDir,
|
|
9
|
+
const { baseDir, hooks } = options;
|
|
10
|
+
const model = await resolveModel(options.model);
|
|
9
11
|
const contentPath = path.join(baseDir, "content");
|
|
10
12
|
const storage = createFileStorage(contentPath);
|
|
11
13
|
const tagSchema = await initializeTagSchema(storage.baseDir, model, options.topic, options.tagSchema);
|
package/dist/greptor.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"greptor.js","sourceRoot":"","sources":["../src/greptor.ts"],"names":[],"mappings":"AAOA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EACN,qBAAqB,EACrB,2BAA2B,EAC3B,sBAAsB,GACtB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAUjE,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAuB;IAC1D,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"greptor.js","sourceRoot":"","sources":["../src/greptor.ts"],"names":[],"mappings":"AAOA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EACN,qBAAqB,EACrB,2BAA2B,EAC3B,sBAAsB,GACtB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAUjE,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAuB;IAC1D,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAE/C,MAAM,SAAS,GAAG,MAAM,mBAAmB,CAC1C,OAAO,CAAC,OAAO,EACf,KAAK,EACL,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,SAAS,CACjB,CAAC;IAEF,MAAM,KAAK,GAAG,qBAAqB,EAAE,CAAC;IACtC,MAAM,WAAW,GAAG,MAAM,2BAA2B,CAAC;QACrD,OAAO;QACP,KAAK;KACL,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG;QACX,MAAM,EAAE,OAAO,CAAC,KAAK;QACrB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;QACpC,KAAK;QACL,OAAO;QACP,KAAK;KACL,CAAC;IAEF,sBAAsB,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC;IAE1E,KAAK,UAAU,GAAG,CAAC,KAAsB;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC7B,KAAK,EAAE,OAAO,EAAE,CAAC;gBAChB,KAAK,EAAE,IAAI,KAAK,CAAC,uBAAuB,KAAK,CAAC,MAAM,EAAE,CAAC;gBACvD,OAAO,EAAE;oBACR,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;iBAClB;aACD,CAAC,CAAC;YACH,OAAO;gBACN,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,uBAAuB,KAAK,CAAC,MAAM,EAAE;aAC9C,CAAC;QACH,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAEhD,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9B,OAAO;gBACN,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,0BAA0B;aACnC,CAAC;QACH,CAAC;QAED,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1B,KAAK,EAAE,OAAO,EAAE,CAAC;gBAChB,KAAK,EAAE,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;gBAC7B,OAAO,EAAE;oBACR,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;iBAClB;aACD,CAAC,CAAC;YACH,OAAO;gBACN,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,GAAG,CAAC,OAAO;aACpB,CAAC;QACH,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEvB,OAAO;YACN,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,gBAAgB;YACzB,GAAG,EAAE,GAAG,CAAC,GAAG;SACZ,CAAC;IACH,CAAC;IAED,KAAK,UAAU,WAAW,CACzB,OAAiB,EACjB,SAAS,GAAG,KAAK;QAEjB,IAAI,CAAC;YACJ,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,aAAa,CACxC;gBACC,MAAM,EAAE,OAAO,CAAC,KAAK;gBACrB,OAAO;gBACP,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,SAAS;gBACT,SAAS;aACT,EACD,OAAO,CACP,CAAC;YAEF,OAAO;gBACN,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,oBAAoB,SAAS,EAAE;gBACxC,SAAS;aACT,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,YAAY,GACjB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxD,KAAK,EAAE,OAAO,EAAE,CAAC;gBAChB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC;aAC/D,CAAC,CAAC;YACH,OAAO;gBACN,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,YAAY;aACrB,CAAC;QACH,CAAC;IACF,CAAC;IAED,OAAO;QACN,GAAG;QACH,WAAW;KACX,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { LanguageModel } from "ai";
|
|
2
|
+
import type { ModelConfig } from "../types.js";
|
|
3
|
+
/**
|
|
4
|
+
* Resolves a LanguageModel from a configuration object.
|
|
5
|
+
*/
|
|
6
|
+
export declare function resolveModel(config: ModelConfig): Promise<LanguageModel>;
|
|
7
|
+
//# sourceMappingURL=llm-factory.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm-factory.d.ts","sourceRoot":"","sources":["../../src/llm/llm-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAqB/C;;GAEG;AACH,wBAAsB,YAAY,CACjC,MAAM,EAAE,WAAW,GACjB,OAAO,CAAC,aAAa,CAAC,CA4CxB"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
const PROVIDER_EXPORTS = {
|
|
2
|
+
"@ai-sdk/amazon-bedrock": "createAmazonBedrock",
|
|
3
|
+
"@ai-sdk/anthropic": "createAnthropic",
|
|
4
|
+
"@ai-sdk/azure": "createAzure",
|
|
5
|
+
"@ai-sdk/cerebras": "createCerebras",
|
|
6
|
+
"@ai-sdk/cohere": "createCohere",
|
|
7
|
+
"@ai-sdk/deepinfra": "createDeepInfra",
|
|
8
|
+
"@ai-sdk/google": "createGoogleGenerativeAI",
|
|
9
|
+
"@ai-sdk/google-vertex": "createVertex",
|
|
10
|
+
"@ai-sdk/groq": "createGroq",
|
|
11
|
+
"@ai-sdk/mistral": "createMistral",
|
|
12
|
+
"@ai-sdk/openai": "createOpenAI",
|
|
13
|
+
"@ai-sdk/openai-compatible": "createOpenAICompatible",
|
|
14
|
+
"@ai-sdk/perplexity": "createPerplexity",
|
|
15
|
+
"@ai-sdk/togetherai": "createTogetherAI",
|
|
16
|
+
"@ai-sdk/xai": "createXai",
|
|
17
|
+
"@openrouter/ai-sdk-provider": "createOpenRouter",
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Resolves a LanguageModel from a configuration object.
|
|
21
|
+
*/
|
|
22
|
+
export async function resolveModel(config) {
|
|
23
|
+
const { provider, model, options } = config;
|
|
24
|
+
const exportName = PROVIDER_EXPORTS[provider];
|
|
25
|
+
if (!exportName) {
|
|
26
|
+
throw new Error(`Unknown or unsupported provider: ${provider}. Please ensure the package is installed.`);
|
|
27
|
+
}
|
|
28
|
+
try {
|
|
29
|
+
// Dynamic import of the provider package
|
|
30
|
+
const mod = await import(provider);
|
|
31
|
+
const factory = mod[exportName];
|
|
32
|
+
if (typeof factory !== "function") {
|
|
33
|
+
throw new Error(`Provider ${provider} does not export a factory function named ${exportName}.`);
|
|
34
|
+
}
|
|
35
|
+
// Create the provider instance (e.g., const openai = createOpenAI({...}))
|
|
36
|
+
const providerInstance = factory(options);
|
|
37
|
+
// The provider instance is typically a function that takes the model ID
|
|
38
|
+
// e.g. openai('gpt-4')
|
|
39
|
+
if (typeof providerInstance !== "function") {
|
|
40
|
+
throw new Error(`Provider factory for ${provider} did not return a valid provider function.`);
|
|
41
|
+
}
|
|
42
|
+
return providerInstance(model);
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
if (error instanceof Error &&
|
|
46
|
+
error.message.includes("Cannot find module")) {
|
|
47
|
+
throw new Error(`Failed to import provider ${provider}. Make sure it is installed in your project.\nrun: bun add ${provider}`);
|
|
48
|
+
}
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=llm-factory.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm-factory.js","sourceRoot":"","sources":["../../src/llm/llm-factory.ts"],"names":[],"mappings":"AAGA,MAAM,gBAAgB,GAA2B;IAChD,wBAAwB,EAAE,qBAAqB;IAC/C,mBAAmB,EAAE,iBAAiB;IACtC,eAAe,EAAE,aAAa;IAC9B,kBAAkB,EAAE,gBAAgB;IACpC,gBAAgB,EAAE,cAAc;IAChC,mBAAmB,EAAE,iBAAiB;IACtC,gBAAgB,EAAE,0BAA0B;IAC5C,uBAAuB,EAAE,cAAc;IACvC,cAAc,EAAE,YAAY;IAC5B,iBAAiB,EAAE,eAAe;IAClC,gBAAgB,EAAE,cAAc;IAChC,2BAA2B,EAAE,wBAAwB;IACrD,oBAAoB,EAAE,kBAAkB;IACxC,oBAAoB,EAAE,kBAAkB;IACxC,aAAa,EAAE,WAAW;IAC1B,6BAA6B,EAAE,kBAAkB;CACjD,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,MAAmB;IAEnB,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAE5C,MAAM,UAAU,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC9C,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CACd,oCAAoC,QAAQ,2CAA2C,CACvF,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACJ,yCAAyC;QACzC,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC;QAEhC,IAAI,OAAO,OAAO,KAAK,UAAU,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CACd,YAAY,QAAQ,6CAA6C,UAAU,GAAG,CAC9E,CAAC;QACH,CAAC;QAED,0EAA0E;QAC1E,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QAE1C,wEAAwE;QACxE,uBAAuB;QACvB,IAAI,OAAO,gBAAgB,KAAK,UAAU,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CACd,wBAAwB,QAAQ,4CAA4C,CAC5E,CAAC;QACH,CAAC;QAED,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IACC,KAAK,YAAY,KAAK;YACtB,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAC3C,CAAC;YACF,MAAM,IAAI,KAAK,CACd,6BAA6B,QAAQ,8DAA8D,QAAQ,EAAE,CAC7G,CAAC;QACH,CAAC;QACD,MAAM,KAAK,CAAC;IACb,CAAC;AACF,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor.d.ts","sourceRoot":"","sources":["../../src/processing/processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAyC,MAAM,IAAI,CAAC;AAE/E,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAQ,MAAM,aAAa,CAAC;AAItD,MAAM,WAAW,gBAAgB;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;CACjC;AAED,MAAM,WAAW,eAAe;IAC/B,OAAO,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IACpC,OAAO,EAAE,MAAM,WAAW,GAAG,SAAS,CAAC;IACvC,IAAI,EAAE,MAAM,MAAM,CAAC;CACnB;AAED,wBAAgB,qBAAqB,IAAI,eAAe,CAgBvD;
|
|
1
|
+
{"version":3,"file":"processor.d.ts","sourceRoot":"","sources":["../../src/processing/processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAyC,MAAM,IAAI,CAAC;AAE/E,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAQ,MAAM,aAAa,CAAC;AAItD,MAAM,WAAW,gBAAgB;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;CACjC;AAED,MAAM,WAAW,eAAe;IAC/B,OAAO,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IACpC,OAAO,EAAE,MAAM,WAAW,GAAG,SAAS,CAAC;IACvC,IAAI,EAAE,MAAM,MAAM,CAAC;CACnB;AAED,wBAAgB,qBAAqB,IAAI,eAAe,CAgBvD;AAoJD,wBAAgB,sBAAsB,CAAC,IAAI,EAAE;IAC5C,GAAG,EAAE,gBAAgB,CAAC;IACtB,KAAK,EAAE,eAAe,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,IAAI,CAwIP;AAED,wBAAsB,2BAA2B,CAAC,IAAI,EAAE;IACvD,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,EAAE,eAAe,CAAC;CACvB,GAAG,OAAO,CAAC,MAAM,CAAC,CAQlB"}
|
|
@@ -44,7 +44,6 @@ Optimize for **single-pass grep scanning**: a single grep hit should reveal what
|
|
|
44
44
|
|
|
45
45
|
## Output Format (Markdown only)
|
|
46
46
|
|
|
47
|
-
\`\`\`md
|
|
48
47
|
## 01 Short descriptive title for chunk 1
|
|
49
48
|
field_1=value_1,value_4
|
|
50
49
|
field_2=value_2,
|
|
@@ -56,7 +55,6 @@ field_1=value_1
|
|
|
56
55
|
field_4=value_4
|
|
57
56
|
field_5=value_5,value_6
|
|
58
57
|
<cleaned, condensed content>
|
|
59
|
-
\`\`\`
|
|
60
58
|
|
|
61
59
|
## Tagging Rules
|
|
62
60
|
- Use ONLY fields defined in the SCHEMA (field names must exactly match schema).
|
|
@@ -89,9 +87,9 @@ ${tagSchema}
|
|
|
89
87
|
${rawContent}
|
|
90
88
|
`;
|
|
91
89
|
}
|
|
92
|
-
async function processDocument(ref, ctx) {
|
|
90
|
+
async function processDocument(ref, ctx, raw) {
|
|
93
91
|
// 1. Read raw content
|
|
94
|
-
const { tags, content } = await ctx.storage.readRawContent(ref);
|
|
92
|
+
const { tags, content } = raw ?? (await ctx.storage.readRawContent(ref));
|
|
95
93
|
// 2. Clean + chunk + tag with a single LLM call
|
|
96
94
|
const prompt = createProcessingPrompt(content, ctx.domain, ctx.tagSchema);
|
|
97
95
|
const { text, usage } = await generateText({
|
|
@@ -107,6 +105,26 @@ async function processDocument(ref, ctx) {
|
|
|
107
105
|
await ctx.storage.saveProcessedContent(ref, rendered);
|
|
108
106
|
return usage;
|
|
109
107
|
}
|
|
108
|
+
function resolveDocumentMetadata(ref, tags) {
|
|
109
|
+
const refParts = ref.split("/");
|
|
110
|
+
const filename = refParts[refParts.length - 1] ?? "";
|
|
111
|
+
let source = refParts[0] ?? "";
|
|
112
|
+
let publisher = refParts.length > 3 ? refParts[1] : undefined;
|
|
113
|
+
let label = filename.replace(/\.md$/, "");
|
|
114
|
+
const tagSource = tags?.source;
|
|
115
|
+
const tagPublisher = tags?.publisher;
|
|
116
|
+
const tagTitle = tags?.title;
|
|
117
|
+
if (typeof tagSource === "string" && tagSource.trim()) {
|
|
118
|
+
source = tagSource;
|
|
119
|
+
}
|
|
120
|
+
if (typeof tagPublisher === "string" && tagPublisher.trim()) {
|
|
121
|
+
publisher = tagPublisher;
|
|
122
|
+
}
|
|
123
|
+
if (typeof tagTitle === "string" && tagTitle.trim()) {
|
|
124
|
+
label = tagTitle;
|
|
125
|
+
}
|
|
126
|
+
return { source, publisher, label };
|
|
127
|
+
}
|
|
110
128
|
function sleep(ms) {
|
|
111
129
|
return new Promise((resolve) => {
|
|
112
130
|
const t = setTimeout(resolve, ms);
|
|
@@ -121,14 +139,18 @@ export function startBackgroundWorkers(args) {
|
|
|
121
139
|
// Shared run state across workers
|
|
122
140
|
let runActive = false;
|
|
123
141
|
let runStartTime = 0;
|
|
124
|
-
let queueSize = 0;
|
|
125
142
|
let runSuccessCount = 0;
|
|
126
143
|
let runFailureCount = 0;
|
|
127
144
|
let runInFlightCount = 0;
|
|
145
|
+
function getQueueTotals() {
|
|
146
|
+
const processed = runActive ? runSuccessCount + runFailureCount : 0;
|
|
147
|
+
const pending = queue.size();
|
|
148
|
+
const total = processed + pending + runInFlightCount;
|
|
149
|
+
return { processed, pending, total };
|
|
150
|
+
}
|
|
128
151
|
function startRun(totalDocs) {
|
|
129
152
|
runActive = true;
|
|
130
153
|
runStartTime = Date.now();
|
|
131
|
-
queueSize = totalDocs;
|
|
132
154
|
runSuccessCount = 0;
|
|
133
155
|
runFailureCount = 0;
|
|
134
156
|
ctx.hooks?.onProcessingRunStarted?.({
|
|
@@ -155,10 +177,10 @@ export function startBackgroundWorkers(args) {
|
|
|
155
177
|
}
|
|
156
178
|
async function workerLoop() {
|
|
157
179
|
while (true) {
|
|
158
|
-
|
|
180
|
+
const { total } = getQueueTotals();
|
|
159
181
|
// Start a new run if there are items and no run is active
|
|
160
|
-
if (
|
|
161
|
-
startRun(
|
|
182
|
+
if (total > 0 && !runActive) {
|
|
183
|
+
startRun(total);
|
|
162
184
|
}
|
|
163
185
|
const docRef = queue.dequeue();
|
|
164
186
|
if (!docRef) {
|
|
@@ -168,25 +190,32 @@ export function startBackgroundWorkers(args) {
|
|
|
168
190
|
continue;
|
|
169
191
|
}
|
|
170
192
|
runInFlightCount++;
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
193
|
+
let raw;
|
|
194
|
+
let readError;
|
|
195
|
+
try {
|
|
196
|
+
raw = await ctx.storage.readRawContent(docRef);
|
|
197
|
+
}
|
|
198
|
+
catch (error) {
|
|
199
|
+
readError = error instanceof Error ? error : new Error(String(error));
|
|
200
|
+
}
|
|
201
|
+
const { source, publisher, label } = resolveDocumentMetadata(docRef, raw?.tags);
|
|
177
202
|
const docStartTime = Date.now();
|
|
203
|
+
const { total: startTotal } = getQueueTotals();
|
|
178
204
|
ctx.hooks?.onDocumentProcessingStarted?.({
|
|
179
205
|
source,
|
|
180
206
|
publisher,
|
|
181
207
|
label,
|
|
182
208
|
successful: runSuccessCount,
|
|
183
209
|
failed: runFailureCount,
|
|
184
|
-
queueSize:
|
|
210
|
+
queueSize: startTotal,
|
|
185
211
|
});
|
|
186
212
|
let usage;
|
|
187
213
|
let success = false;
|
|
188
214
|
try {
|
|
189
|
-
|
|
215
|
+
if (readError) {
|
|
216
|
+
throw readError;
|
|
217
|
+
}
|
|
218
|
+
usage = await processDocument(docRef, ctx, raw);
|
|
190
219
|
runSuccessCount++;
|
|
191
220
|
success = true;
|
|
192
221
|
}
|
|
@@ -200,6 +229,7 @@ export function startBackgroundWorkers(args) {
|
|
|
200
229
|
finally {
|
|
201
230
|
runInFlightCount--;
|
|
202
231
|
}
|
|
232
|
+
const { total: endTotal } = getQueueTotals();
|
|
203
233
|
ctx.hooks?.onDocumentProcessingCompleted?.({
|
|
204
234
|
success,
|
|
205
235
|
source,
|
|
@@ -207,7 +237,7 @@ export function startBackgroundWorkers(args) {
|
|
|
207
237
|
label,
|
|
208
238
|
successful: runSuccessCount,
|
|
209
239
|
failed: runFailureCount,
|
|
210
|
-
queueSize:
|
|
240
|
+
queueSize: endTotal,
|
|
211
241
|
elapsedMs: Date.now() - docStartTime,
|
|
212
242
|
inputTokens: usage?.inputTokens ?? 0,
|
|
213
243
|
outputTokens: usage?.outputTokens ?? 0,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor.js","sourceRoot":"","sources":["../../src/processing/processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAA+C,YAAY,EAAE,MAAM,IAAI,CAAC;AAC/E,OAAO,IAAI,MAAM,MAAM,CAAC;AAIxB,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAgBlC,MAAM,UAAU,qBAAqB;IACpC,MAAM,KAAK,GAAkB,EAAE,CAAC;IAEhC,OAAO;QACN,OAAO,CAAC,GAAG;YACV,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;QAED,IAAI;YACH,OAAO,KAAK,CAAC,MAAM,CAAC;QACrB,CAAC;QAED,OAAO;YACN,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;KACD,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAU,EAAE,YAAoB;IAChE,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEpC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE;QACf,GAAG,CAAC,CAAC,EAAE,IAAI;YACV,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;YACnE,IAAI,UAAU,EAAE,CAAC;gBAChB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YAClB,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,GAAG,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IAEtD,OAAO,CAAC,KAAK,EAAE,YAAY,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAC1E,IAAI,CACJ,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAC9B,UAAkB,EAClB,MAAc,EACd,SAAiB;IAEjB,OAAO;;iFAEyE,MAAM
|
|
1
|
+
{"version":3,"file":"processor.js","sourceRoot":"","sources":["../../src/processing/processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAA+C,YAAY,EAAE,MAAM,IAAI,CAAC;AAC/E,OAAO,IAAI,MAAM,MAAM,CAAC;AAIxB,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAgBlC,MAAM,UAAU,qBAAqB;IACpC,MAAM,KAAK,GAAkB,EAAE,CAAC;IAEhC,OAAO;QACN,OAAO,CAAC,GAAG;YACV,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;QAED,IAAI;YACH,OAAO,KAAK,CAAC,MAAM,CAAC;QACrB,CAAC;QAED,OAAO;YACN,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;KACD,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAU,EAAE,YAAoB;IAChE,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEpC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE;QACf,GAAG,CAAC,CAAC,EAAE,IAAI;YACV,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;YACnE,IAAI,UAAU,EAAE,CAAC;gBAChB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YAClB,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,GAAG,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IAEtD,OAAO,CAAC,KAAK,EAAE,YAAY,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAC1E,IAAI,CACJ,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAC9B,UAAkB,EAClB,MAAc,EACd,SAAiB;IAEjB,OAAO;;iFAEyE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAkDrF,SAAS;;;EAGT,UAAU;CACX,CAAC;AACF,CAAC;AAED,KAAK,UAAU,eAAe,CAC7B,GAAgB,EAChB,GAAqB,EACrB,GAAqC;IAErC,sBAAsB;IACtB,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC;IAEzE,gDAAgD;IAChD,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;IAE1E,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,YAAY,CAAC;QAC1C,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,MAAM;KACN,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IAClE,CAAC;IAED,wDAAwD;IACxD,MAAM,QAAQ,GAAG,uBAAuB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAErD,4BAA4B;IAC5B,MAAM,GAAG,CAAC,OAAO,CAAC,oBAAoB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC;AACd,CAAC;AAED,SAAS,uBAAuB,CAC/B,GAAgB,EAChB,IAAW;IAEX,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAChC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACrD,IAAI,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/B,IAAI,SAAS,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9D,IAAI,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAE1C,MAAM,SAAS,GAAG,IAAI,EAAE,MAAM,CAAC;IAC/B,MAAM,YAAY,GAAG,IAAI,EAAE,SAAS,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,EAAE,KAAK,CAAC;IAE7B,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;QACvD,MAAM,GAAG,SAAS,CAAC;IACpB,CAAC;IACD,IAAI,OAAO,YAAY,KAAK,QAAQ,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;QAC7D,SAAS,GAAG,YAAY,CAAC;IAC1B,CAAC;IACD,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACrD,KAAK,GAAG,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACxB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC9B,MAAM,CAAC,GAAG,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,kEAAkE;QACjE,CAAuC,CAAC,KAAK,EAAE,EAAE,CAAC;IACpD,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,IAKtC;IACA,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IACvD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,WAAW,IAAI,qBAAqB,CAAC,CAAC;IAC5E,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IAE5B,kCAAkC;IAClC,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IAEzB,SAAS,cAAc;QACtB,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,eAAe,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,SAAS,GAAG,OAAO,GAAG,gBAAgB,CAAC;QACrD,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACtC,CAAC;IAED,SAAS,QAAQ,CAAC,SAAiB;QAClC,SAAS,GAAG,IAAI,CAAC;QACjB,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC1B,eAAe,GAAG,CAAC,CAAC;QACpB,eAAe,GAAG,CAAC,CAAC;QAEpB,GAAG,CAAC,KAAK,EAAE,sBAAsB,EAAE,CAAC;YACnC,kBAAkB,EAAE,SAAS;YAC7B,cAAc,EAAE,SAAS;SACzB,CAAC,CAAC;IACJ,CAAC;IAED,SAAS,MAAM;QACd,IAAI,CAAC,SAAS;YAAE,OAAO;QACvB,SAAS,GAAG,KAAK,CAAC;QAElB,GAAG,CAAC,KAAK,EAAE,wBAAwB,EAAE,CAAC;YACrC,UAAU,EAAE,eAAe;YAC3B,MAAM,EAAE,eAAe;YACvB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY;SACpC,CAAC,CAAC;IACJ,CAAC;IAED,SAAS,SAAS;QACjB,IAAI,CAAC,SAAS;YAAE,OAAO;QACvB,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,gBAAgB,KAAK,CAAC,EAAE,CAAC;YAClD,MAAM,EAAE,CAAC;QACV,CAAC;IACF,CAAC;IAED,KAAK,UAAU,UAAU;QACxB,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,EAAE,KAAK,EAAE,GAAG,cAAc,EAAE,CAAC;YAEnC,0DAA0D;YAC1D,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC7B,QAAQ,CAAC,KAAK,CAAC,CAAC;YACjB,CAAC;YAED,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,qDAAqD;gBACrD,SAAS,EAAE,CAAC;gBACZ,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC;gBACzB,SAAS;YACV,CAAC;YAED,gBAAgB,EAAE,CAAC;YAEnB,IAAI,GAAgD,CAAC;YACrD,IAAI,SAA4B,CAAC;YAEjC,IAAI,CAAC;gBACJ,GAAG,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACvE,CAAC;YAED,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,uBAAuB,CAC3D,MAAM,EACN,GAAG,EAAE,IAAI,CACT,CAAC;YAEF,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAChC,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,cAAc,EAAE,CAAC;YAE/C,GAAG,CAAC,KAAK,EAAE,2BAA2B,EAAE,CAAC;gBACxC,MAAM;gBACN,SAAS;gBACT,KAAK;gBACL,UAAU,EAAE,eAAe;gBAC3B,MAAM,EAAE,eAAe;gBACvB,SAAS,EAAE,UAAU;aACrB,CAAC,CAAC;YAEH,IAAI,KAAqC,CAAC;YAC1C,IAAI,OAAO,GAAG,KAAK,CAAC;YAEpB,IAAI,CAAC;gBACJ,IAAI,SAAS,EAAE,CAAC;oBACf,MAAM,SAAS,CAAC;gBACjB,CAAC;gBACD,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;gBAChD,eAAe,EAAE,CAAC;gBAClB,OAAO,GAAG,IAAI,CAAC;YAChB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,eAAe,EAAE,CAAC;gBAClB,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC;oBACpB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAChE,OAAO,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE;iBAClD,CAAC,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACV,gBAAgB,EAAE,CAAC;YACpB,CAAC;YAED,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,cAAc,EAAE,CAAC;YAC7C,GAAG,CAAC,KAAK,EAAE,6BAA6B,EAAE,CAAC;gBAC1C,OAAO;gBACP,MAAM;gBACN,SAAS;gBACT,KAAK;gBACL,UAAU,EAAE,eAAe;gBAC3B,MAAM,EAAE,eAAe;gBACvB,SAAS,EAAE,QAAQ;gBACnB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY;gBACpC,WAAW,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;gBACpC,YAAY,EAAE,KAAK,EAAE,YAAY,IAAI,CAAC;gBACtC,WAAW,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;aACpC,CAAC,CAAC;YAEH,SAAS,EAAE,CAAC;QACb,CAAC;IACF,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,UAAU,EAAE,CAAC;IACd,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,IAGjD;IACA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;IAEzD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,OAAO,IAAI,CAAC,MAAM,CAAC;AACpB,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { LanguageModel } from "ai";
|
|
2
1
|
import type { DocumentRef } from "./storage/types.js";
|
|
3
2
|
import type { TagSchema } from "./tag-schema/types.js";
|
|
4
3
|
export type { TagSchema, TagSchemaItem, } from "./tag-schema/types.js";
|
|
@@ -54,10 +53,30 @@ export interface GreptorHooks {
|
|
|
54
53
|
onDocumentProcessingCompleted?: (event: DocumentProcessingCompletedEvent) => void;
|
|
55
54
|
onError?: (event: ErrorEvent) => void;
|
|
56
55
|
}
|
|
56
|
+
export interface ModelConfig {
|
|
57
|
+
/**
|
|
58
|
+
* The name of the AI SDK provider fro [AI SDK ecosystem](https://sdk.vercel.ai/providers/ai-sdk-providers):
|
|
59
|
+
- `@ai-sdk/openai` - OpenAI (GPT-4, GPT-4o, etc.)
|
|
60
|
+
- `@ai-sdk/anthropic` - Anthropic (Claude)
|
|
61
|
+
- `@ai-sdk/groq` - Groq (fast inference)
|
|
62
|
+
- `@ai-sdk/openai-compatible` - OpenAI-compatible endpoints (NVIDIA NIM, OpenRouter, etc.)
|
|
63
|
+
- And [many more](https://sdk.vercel.ai/providers/ai-sdk-providers)...
|
|
64
|
+
*/
|
|
65
|
+
provider: string;
|
|
66
|
+
/**
|
|
67
|
+
* The model ID to use (specific to the provider).
|
|
68
|
+
*/
|
|
69
|
+
model: string;
|
|
70
|
+
/**
|
|
71
|
+
* Configuration options passed to the provider factory function.
|
|
72
|
+
* Examples: apiKey, baseURL, etc.
|
|
73
|
+
*/
|
|
74
|
+
options?: Record<string, unknown>;
|
|
75
|
+
}
|
|
57
76
|
export interface GreptorOptions {
|
|
58
77
|
baseDir: string;
|
|
59
78
|
topic: string;
|
|
60
|
-
model:
|
|
79
|
+
model: ModelConfig;
|
|
61
80
|
workers?: number;
|
|
62
81
|
tagSchema?: TagSchema;
|
|
63
82
|
hooks?: GreptorHooks;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EACX,SAAS,EACT,aAAa,GACb,MAAM,uBAAuB,CAAC;AAE/B,kDAAkD;AAClD,MAAM,WAAW,yBAAyB;IACzC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;CACvB;AAED,qDAAqD;AACrD,MAAM,WAAW,2BAA2B;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,qDAAqD;AACrD,MAAM,WAAW,8BAA8B;IAC9C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,gCAAgC;IAChD,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACpB;AAED,4BAA4B;AAC5B,MAAM,WAAW,UAAU;IAC1B,KAAK,EAAE,KAAK,CAAC;IACb,OAAO,CAAC,EAAE;QACT,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QAC/B,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QAC3B,GAAG,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;KAC9B,CAAC;CACF;AAED,wCAAwC;AACxC,MAAM,WAAW,YAAY;IAC5B,sBAAsB,CAAC,EAAE,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;IACpE,wBAAwB,CAAC,EAAE,CAAC,KAAK,EAAE,2BAA2B,KAAK,IAAI,CAAC;IACxE,2BAA2B,CAAC,EAAE,CAAC,KAAK,EAAE,8BAA8B,KAAK,IAAI,CAAC;IAC9E,6BAA6B,CAAC,EAAE,CAC/B,KAAK,EAAE,gCAAgC,KACnC,IAAI,CAAC;IACV,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,WAAW;IAC3B;;;;;;;OAOG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,cAAc;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,WAAW,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,KAAK,CAAC,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC;AAErC,MAAM,MAAM,YAAY,GACrB,MAAM,GACN,MAAM,GACN,OAAO,GACP,IAAI,GACJ,MAAM,EAAE,GACR,MAAM,EAAE,GACR,OAAO,EAAE,CAAC;AAEb,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AAEhD,MAAM,WAAW,eAAe;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,IAAI,CAAC;IACpB,IAAI,CAAC,EAAE,IAAI,CAAC;IACZ,SAAS,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,MAAM,gBAAgB,GACzB;IAAE,OAAO,EAAE,IAAI,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,WAAW,CAAA;CAAE,GACpD;IAAE,OAAO,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvC,MAAM,MAAM,iBAAiB,GAC1B;IAAE,OAAO,EAAE,IAAI,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACrD;IAAE,OAAO,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "greptor",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Transform messy, unstructured text into clean, grep-friendly data for agentic search workflows.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -57,6 +57,24 @@
|
|
|
57
57
|
},
|
|
58
58
|
"dependencies": {
|
|
59
59
|
"@ai-sdk/openai-compatible": "^2.0.2",
|
|
60
|
+
"@ai-sdk/amazon-bedrock": "3.0.57",
|
|
61
|
+
"@ai-sdk/anthropic": "2.0.56",
|
|
62
|
+
"@ai-sdk/azure": "2.0.82",
|
|
63
|
+
"@ai-sdk/cerebras": "1.0.33",
|
|
64
|
+
"@ai-sdk/cohere": "2.0.21",
|
|
65
|
+
"@ai-sdk/deepinfra": "1.0.30",
|
|
66
|
+
"@ai-sdk/gateway": "2.0.23",
|
|
67
|
+
"@ai-sdk/google": "2.0.49",
|
|
68
|
+
"@ai-sdk/google-vertex": "3.0.81",
|
|
69
|
+
"@ai-sdk/groq": "2.0.33",
|
|
70
|
+
"@ai-sdk/mistral": "2.0.26",
|
|
71
|
+
"@ai-sdk/openai": "2.0.71",
|
|
72
|
+
"@ai-sdk/perplexity": "2.0.22",
|
|
73
|
+
"@ai-sdk/provider": "2.0.0",
|
|
74
|
+
"@ai-sdk/provider-utils": "3.0.19",
|
|
75
|
+
"@ai-sdk/togetherai": "1.0.30",
|
|
76
|
+
"@ai-sdk/vercel": "1.0.31",
|
|
77
|
+
"@ai-sdk/xai": "2.0.42",
|
|
60
78
|
"ai": "^6.0.6",
|
|
61
79
|
"yaml": "^2.8.2",
|
|
62
80
|
"zod": "^4.1.12"
|