@revealui/ai 0.2.8 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/a2a/card.d.ts +1 -1
- package/dist/a2a/card.d.ts.map +1 -1
- package/dist/a2a/card.js +4 -4
- package/dist/a2a/handler.d.ts +4 -4
- package/dist/a2a/handler.js +5 -5
- package/dist/a2a/index.d.ts +1 -1
- package/dist/a2a/index.js +1 -1
- package/dist/audit/emitter.d.ts +1 -1
- package/dist/audit/emitter.js +2 -2
- package/dist/audit/index.d.ts +2 -2
- package/dist/audit/index.js +2 -2
- package/dist/audit/store.d.ts +2 -2
- package/dist/audit/store.js +2 -2
- package/dist/client/errors.d.ts +13 -0
- package/dist/client/errors.d.ts.map +1 -0
- package/dist/client/errors.js +28 -0
- package/dist/client/hooks/useAgentContext.d.ts.map +1 -1
- package/dist/client/hooks/useAgentContext.js +6 -5
- package/dist/client/hooks/useAgentStream.d.ts +2 -2
- package/dist/client/hooks/useAgentStream.js +3 -3
- package/dist/client/hooks/useEpisodicMemory.d.ts.map +1 -1
- package/dist/client/hooks/useEpisodicMemory.js +6 -5
- package/dist/client/hooks/useWorkingMemory.d.ts.map +1 -1
- package/dist/client/hooks/useWorkingMemory.js +7 -6
- package/dist/embeddings/index.d.ts +1 -1
- package/dist/embeddings/index.js +4 -4
- package/dist/inference/context-budget.d.ts +1 -1
- package/dist/inference/context-budget.js +4 -4
- package/dist/inference/index.d.ts +1 -1
- package/dist/inference/index.js +1 -1
- package/dist/inference/runRag.d.ts +3 -3
- package/dist/inference/runRag.d.ts.map +1 -1
- package/dist/inference/runRag.js +1 -1
- package/dist/inference/task-decomposer.d.ts +1 -1
- package/dist/inference/task-decomposer.d.ts.map +1 -1
- package/dist/inference/task-decomposer.js +3 -3
- package/dist/inference/tool-result-compressor.d.ts +1 -1
- package/dist/inference/tool-result-compressor.js +1 -1
- package/dist/ingestion/{cms-indexer.d.ts → admin-indexer.d.ts} +12 -12
- package/dist/ingestion/admin-indexer.d.ts.map +1 -0
- package/dist/ingestion/{cms-indexer.js → admin-indexer.js} +10 -10
- package/dist/ingestion/bm25.d.ts +1 -1
- package/dist/ingestion/bm25.js +1 -1
- package/dist/ingestion/file-parsers.d.ts +1 -1
- package/dist/ingestion/file-parsers.d.ts.map +1 -1
- package/dist/ingestion/file-parsers.js +36 -17
- package/dist/ingestion/hybrid-search.d.ts +1 -1
- package/dist/ingestion/hybrid-search.js +1 -1
- package/dist/ingestion/index.d.ts +1 -1
- package/dist/ingestion/index.d.ts.map +1 -1
- package/dist/ingestion/index.js +1 -1
- package/dist/ingestion/pipeline.d.ts +1 -1
- package/dist/ingestion/pipeline.d.ts.map +1 -1
- package/dist/ingestion/rag-vector-service.d.ts +1 -1
- package/dist/ingestion/rag-vector-service.js +1 -1
- package/dist/ingestion/reranker.d.ts +1 -1
- package/dist/ingestion/reranker.js +1 -1
- package/dist/ingestion/text-splitter.d.ts +1 -1
- package/dist/ingestion/text-splitter.js +2 -2
- package/dist/llm/client.d.ts +7 -16
- package/dist/llm/client.d.ts.map +1 -1
- package/dist/llm/client.js +10 -70
- package/dist/llm/key-validator.d.ts +1 -1
- package/dist/llm/key-validator.js +8 -8
- package/dist/llm/providers/base.d.ts +2 -2
- package/dist/llm/providers/groq.d.ts +2 -2
- package/dist/llm/providers/groq.d.ts.map +1 -1
- package/dist/llm/providers/groq.js +2 -2
- package/dist/llm/providers/inference-snaps.d.ts +5 -5
- package/dist/llm/providers/inference-snaps.d.ts.map +1 -1
- package/dist/llm/providers/inference-snaps.js +4 -4
- package/dist/llm/providers/ollama.d.ts +2 -2
- package/dist/llm/providers/ollama.d.ts.map +1 -1
- package/dist/llm/providers/ollama.js +1 -1
- package/dist/llm/providers/openai-compat.d.ts +2 -2
- package/dist/llm/providers/openai-compat.js +5 -5
- package/dist/llm/providers/vultr.js +1 -1
- package/dist/llm/server.d.ts +0 -1
- package/dist/llm/server.d.ts.map +1 -1
- package/dist/llm/server.js +0 -1
- package/dist/llm/token-counter.d.ts.map +1 -1
- package/dist/llm/token-counter.js +11 -8
- package/dist/llm/workspace-provider-config.d.ts +1 -1
- package/dist/llm/workspace-provider-config.d.ts.map +1 -1
- package/dist/llm/workspace-provider-config.js +1 -1
- package/dist/memory/crdt/lww-register.js +1 -1
- package/dist/memory/crdt/or-set.d.ts +12 -0
- package/dist/memory/crdt/or-set.d.ts.map +1 -1
- package/dist/memory/crdt/or-set.js +27 -0
- package/dist/memory/index.d.ts +1 -0
- package/dist/memory/index.d.ts.map +1 -1
- package/dist/memory/index.js +1 -0
- package/dist/memory/persistence/crdt-persistence.d.ts +21 -1
- package/dist/memory/persistence/crdt-persistence.d.ts.map +1 -1
- package/dist/memory/persistence/crdt-persistence.js +67 -0
- package/dist/memory/preferences/user-preferences-manager.d.ts.map +1 -1
- package/dist/memory/preferences/user-preferences-manager.js +12 -2
- package/dist/memory/stores/episodic-memory.js +2 -2
- package/dist/memory/stores/working-memory.d.ts +7 -2
- package/dist/memory/stores/working-memory.d.ts.map +1 -1
- package/dist/memory/stores/working-memory.js +32 -18
- package/dist/memory/sync/index.d.ts +2 -0
- package/dist/memory/sync/index.d.ts.map +1 -0
- package/dist/memory/sync/index.js +1 -0
- package/dist/memory/sync/sync-manager.d.ts +104 -0
- package/dist/memory/sync/sync-manager.d.ts.map +1 -0
- package/dist/memory/sync/sync-manager.js +137 -0
- package/dist/memory/utils/index.d.ts +1 -2
- package/dist/memory/utils/index.d.ts.map +1 -1
- package/dist/memory/utils/index.js +1 -2
- package/dist/memory/utils/sql-helpers.d.ts +0 -39
- package/dist/memory/utils/sql-helpers.d.ts.map +1 -1
- package/dist/memory/utils/sql-helpers.js +0 -88
- package/dist/memory/utils/validation.js +1 -1
- package/dist/memory/vector/vector-memory-service.d.ts +1 -1
- package/dist/memory/vector/vector-memory-service.js +1 -1
- package/dist/orchestration/agent.d.ts +2 -2
- package/dist/orchestration/defaults.d.ts +1 -1
- package/dist/orchestration/defaults.js +1 -1
- package/dist/orchestration/orchestrator.d.ts +3 -3
- package/dist/orchestration/orchestrator.js +3 -3
- package/dist/orchestration/runtime.d.ts +1 -1
- package/dist/orchestration/runtime.js +1 -1
- package/dist/orchestration/streaming-runtime.d.ts +2 -2
- package/dist/orchestration/streaming-runtime.js +2 -2
- package/dist/orchestration/ticket-agent.d.ts +11 -11
- package/dist/orchestration/ticket-agent.d.ts.map +1 -1
- package/dist/orchestration/ticket-agent.js +10 -10
- package/dist/skills/catalog/vercel-catalog.d.ts.map +1 -1
- package/dist/skills/catalog/vercel-catalog.js +16 -7
- package/dist/skills/loader/github-loader.d.ts.map +1 -1
- package/dist/skills/loader/github-loader.js +2 -0
- package/dist/skills/loader/local-loader.js +1 -1
- package/dist/skills/loader/vercel-loader.d.ts.map +1 -1
- package/dist/skills/loader/vercel-loader.js +2 -0
- package/dist/skills/parser/skill-md-parser.js +2 -2
- package/dist/skills/registry/skill-registry.js +1 -1
- package/dist/templates/prompt-spec.js +1 -1
- package/dist/templates/skill-spec.js +1 -1
- package/dist/tools/{cms → admin}/collection-tools.d.ts +2 -2
- package/dist/tools/admin/collection-tools.d.ts.map +1 -0
- package/dist/tools/{cms → admin}/collection-tools.js +8 -8
- package/dist/tools/{cms → admin}/factory.d.ts +11 -11
- package/dist/tools/admin/factory.d.ts.map +1 -0
- package/dist/tools/{cms → admin}/factory.js +4 -4
- package/dist/tools/{cms → admin}/global-tools.d.ts +1 -1
- package/dist/tools/admin/global-tools.d.ts.map +1 -0
- package/dist/tools/{cms → admin}/global-tools.js +4 -4
- package/dist/tools/{cms → admin}/index.d.ts +4 -4
- package/dist/tools/admin/index.d.ts.map +1 -0
- package/dist/tools/{cms → admin}/index.js +3 -3
- package/dist/tools/{cms → admin}/media-tools.d.ts +1 -1
- package/dist/tools/admin/media-tools.d.ts.map +1 -0
- package/dist/tools/{cms → admin}/media-tools.js +4 -4
- package/dist/tools/{cms → admin}/user-tools.d.ts +1 -1
- package/dist/tools/admin/user-tools.d.ts.map +1 -0
- package/dist/tools/{cms → admin}/user-tools.js +1 -1
- package/dist/tools/coding/file-edit.d.ts +1 -1
- package/dist/tools/coding/file-edit.js +2 -2
- package/dist/tools/coding/file-glob.d.ts +1 -1
- package/dist/tools/coding/file-glob.d.ts.map +1 -1
- package/dist/tools/coding/file-glob.js +2 -1
- package/dist/tools/coding/file-grep.d.ts +1 -1
- package/dist/tools/coding/file-grep.d.ts.map +1 -1
- package/dist/tools/coding/file-grep.js +2 -1
- package/dist/tools/coding/file-read.d.ts +1 -1
- package/dist/tools/coding/file-read.d.ts.map +1 -1
- package/dist/tools/coding/file-read.js +15 -9
- package/dist/tools/coding/file-write.d.ts +1 -1
- package/dist/tools/coding/file-write.js +1 -1
- package/dist/tools/coding/git-ops.d.ts +1 -1
- package/dist/tools/coding/git-ops.d.ts.map +1 -1
- package/dist/tools/coding/git-ops.js +5 -7
- package/dist/tools/coding/index.d.ts +1 -1
- package/dist/tools/coding/index.d.ts.map +1 -1
- package/dist/tools/coding/lint-fix.d.ts +1 -1
- package/dist/tools/coding/lint-fix.d.ts.map +1 -1
- package/dist/tools/coding/lint-fix.js +8 -4
- package/dist/tools/coding/project-context.d.ts +1 -1
- package/dist/tools/coding/project-context.d.ts.map +1 -1
- package/dist/tools/coding/project-context.js +25 -24
- package/dist/tools/coding/safety.d.ts +1 -1
- package/dist/tools/coding/safety.d.ts.map +1 -1
- package/dist/tools/coding/shell-exec.d.ts +1 -1
- package/dist/tools/coding/shell-exec.js +1 -1
- package/dist/tools/coding/test-runner.d.ts +1 -1
- package/dist/tools/coding/test-runner.d.ts.map +1 -1
- package/dist/tools/coding/test-runner.js +12 -7
- package/dist/tools/deduplicator.js +1 -1
- package/dist/tools/document-summarizer.js +2 -2
- package/dist/tools/memory/store-memory.d.ts +1 -1
- package/dist/tools/memory/store-memory.js +2 -2
- package/dist/tools/ticket-tools.d.ts +2 -2
- package/dist/tools/ticket-tools.js +3 -3
- package/dist/tools/web/duck-duck-go.d.ts +3 -3
- package/dist/tools/web/duck-duck-go.js +4 -4
- package/dist/tools/web/exa.d.ts +1 -1
- package/dist/tools/web/exa.js +1 -1
- package/dist/tools/web/scraper.js +1 -1
- package/dist/tools/web/tavily.d.ts +2 -2
- package/dist/tools/web/tavily.js +2 -2
- package/dist/tools/web/types.d.ts +2 -2
- package/dist/tools/web/types.js +2 -2
- package/package.json +44 -15
- package/dist/ingestion/cms-indexer.d.ts.map +0 -1
- package/dist/llm/providers/bitnet.d.ts +0 -28
- package/dist/llm/providers/bitnet.d.ts.map +0 -1
- package/dist/llm/providers/bitnet.js +0 -36
- package/dist/tools/cms/collection-tools.d.ts.map +0 -1
- package/dist/tools/cms/factory.d.ts.map +0 -1
- package/dist/tools/cms/global-tools.d.ts.map +0 -1
- package/dist/tools/cms/index.d.ts.map +0 -1
- package/dist/tools/cms/media-tools.d.ts.map +0 -1
- package/dist/tools/cms/user-tools.d.ts.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runRag.d.ts","sourceRoot":"","sources":["../../src/inference/runRag.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAIlD,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,EAAE,EAAE,QAAQ,CAAC;IACb,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,
|
|
1
|
+
{"version":3,"file":"runRag.d.ts","sourceRoot":"","sources":["../../src/inference/runRag.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAIlD,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,EAAE,EAAE,QAAQ,CAAC;IACb,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,sFAAsF;IACtF,IAAI,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IAC5B,qEAAqE;IACrE,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,0DAA0D;IAC1D,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,4CAA4C;IAC5C,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,wBAAsB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAyCjF"}
|
package/dist/inference/runRag.js
CHANGED
|
@@ -48,7 +48,7 @@ export interface DecomposerConfig {
|
|
|
48
48
|
maxSteps: number;
|
|
49
49
|
/** Model name for tier classification */
|
|
50
50
|
modelName: string;
|
|
51
|
-
/** Min complexity threshold
|
|
51
|
+
/** Min complexity threshold - skip decomposition for simple tasks (default: 1) */
|
|
52
52
|
minComplexityForDecomposition: number;
|
|
53
53
|
}
|
|
54
54
|
export declare function configureDecomposer(overrides: Partial<DecomposerConfig>): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"task-decomposer.d.ts","sourceRoot":"","sources":["../../src/inference/task-decomposer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAmC,KAAK,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAMtF,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,wCAAwC;IACxC,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC,sDAAsD;IACtD,UAAU,EAAE,OAAO,CAAC;IACpB,2BAA2B;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,wBAAwB;IACxB,QAAQ,CAAC,EAAE;QACT,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,SAAS,CAAC,EAAE,SAAS,CAAC;KACvB,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;IAClB,
|
|
1
|
+
{"version":3,"file":"task-decomposer.d.ts","sourceRoot":"","sources":["../../src/inference/task-decomposer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAmC,KAAK,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAMtF,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,wCAAwC;IACxC,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC,sDAAsD;IACtD,UAAU,EAAE,OAAO,CAAC;IACpB,2BAA2B;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,wBAAwB;IACxB,QAAQ,CAAC,EAAE;QACT,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,SAAS,CAAC,EAAE,SAAS,CAAC;KACvB,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;IAClB,oFAAoF;IACpF,6BAA6B,EAAE,MAAM,CAAC;CACvC;AAUD,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAE9E;AAkBD;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAmB9D;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,GAAG,OAAO,CAWlF;AAwBD;;;;GAIG;AACH,wBAAsB,aAAa,CACjC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,SAAS,EACpB,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,mBAAmB,CAAC,CAuD9B;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,CAQjF"}
|
|
@@ -54,7 +54,7 @@ export function estimateComplexity(instruction) {
|
|
|
54
54
|
if (words > 100)
|
|
55
55
|
score += 1;
|
|
56
56
|
// Path/file references suggest multi-file work
|
|
57
|
-
const pathRefs = instruction.match(/[\w
|
|
57
|
+
const pathRefs = instruction.match(/[\w/]+(?:[.-][\w/]+)*\.\w{1,5}/g) ?? [];
|
|
58
58
|
if (pathRefs.length > 1)
|
|
59
59
|
score += pathRefs.length * 0.5;
|
|
60
60
|
return Math.min(10, score);
|
|
@@ -111,7 +111,7 @@ export async function decomposeTask(instruction, llmClient, modelName) {
|
|
|
111
111
|
};
|
|
112
112
|
}
|
|
113
113
|
try {
|
|
114
|
-
// Use a focused planning prompt
|
|
114
|
+
// Use a focused planning prompt - keep it small for small models
|
|
115
115
|
const planningInstruction = `${PLANNING_PROMPT}${instruction}`;
|
|
116
116
|
const budget = getContextBudget(tier);
|
|
117
117
|
// For planning, we want minimal context
|
|
@@ -140,7 +140,7 @@ export async function decomposeTask(instruction, llmClient, modelName) {
|
|
|
140
140
|
};
|
|
141
141
|
}
|
|
142
142
|
catch {
|
|
143
|
-
// Decomposition failed
|
|
143
|
+
// Decomposition failed - fall back to single-step
|
|
144
144
|
return {
|
|
145
145
|
decomposed: false,
|
|
146
146
|
original: instruction,
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Deterministic, per-tool-type compression of tool results based on model tier.
|
|
5
5
|
* Small models get aggressively truncated output; large models get full output.
|
|
6
6
|
*
|
|
7
|
-
* No LLM-based summarization
|
|
7
|
+
* No LLM-based summarization - all compression is structural (head/tail,
|
|
8
8
|
* match limits, line limits) so it's fast and predictable.
|
|
9
9
|
*/
|
|
10
10
|
import type { ModelTier } from './context-budget.js';
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Deterministic, per-tool-type compression of tool results based on model tier.
|
|
5
5
|
* Small models get aggressively truncated output; large models get full output.
|
|
6
6
|
*
|
|
7
|
-
* No LLM-based summarization
|
|
7
|
+
* No LLM-based summarization - all compression is structural (head/tail,
|
|
8
8
|
* match limits, line limits) so it's fast and predictable.
|
|
9
9
|
*/
|
|
10
10
|
const DEFAULT_LIMITS = {
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* admin Indexer
|
|
3
3
|
*
|
|
4
|
-
* Handles automatic re-indexing of
|
|
5
|
-
* Wire into
|
|
4
|
+
* Handles automatic re-indexing of admin documents when they change.
|
|
5
|
+
* Wire into admin collection afterChange hooks - no admin API calls from here,
|
|
6
6
|
* the event payload carries the document content directly.
|
|
7
7
|
*
|
|
8
|
-
* Usage (in apps/
|
|
9
|
-
* export const
|
|
8
|
+
* Usage (in apps/admin/src/lib/ai/indexer.ts):
|
|
9
|
+
* export const adminIndexer = new AdminIndexer({ db, ingestionPipeline, enabledCollections: ['posts', 'pages'] })
|
|
10
10
|
*
|
|
11
|
-
* In each
|
|
12
|
-
* await
|
|
11
|
+
* In each admin collection afterChange hook:
|
|
12
|
+
* await adminIndexer.onDocumentChanged({ collection: 'posts', id: doc.id, operation, doc })
|
|
13
13
|
*/
|
|
14
14
|
import type { IngestionPipeline } from './pipeline.js';
|
|
15
15
|
export interface CmsDocumentEvent {
|
|
@@ -19,21 +19,21 @@ export interface CmsDocumentEvent {
|
|
|
19
19
|
doc?: Record<string, unknown>;
|
|
20
20
|
workspaceId?: string;
|
|
21
21
|
}
|
|
22
|
-
export interface
|
|
22
|
+
export interface AdminIndexerConfig {
|
|
23
23
|
ingestionPipeline: IngestionPipeline;
|
|
24
24
|
enabledCollections: string[];
|
|
25
25
|
/** Default workspaceId when not provided per-event */
|
|
26
26
|
defaultWorkspaceId?: string;
|
|
27
27
|
}
|
|
28
|
-
export declare class
|
|
28
|
+
export declare class AdminIndexer {
|
|
29
29
|
private pipeline;
|
|
30
30
|
private enabledCollections;
|
|
31
31
|
private defaultWorkspaceId;
|
|
32
|
-
constructor(config:
|
|
32
|
+
constructor(config: AdminIndexerConfig);
|
|
33
33
|
/**
|
|
34
|
-
* Handle
|
|
34
|
+
* Handle an admin document change event.
|
|
35
35
|
* Skips collections not in enabledCollections.
|
|
36
36
|
*/
|
|
37
37
|
onDocumentChanged(event: CmsDocumentEvent): Promise<void>;
|
|
38
38
|
}
|
|
39
|
-
//# sourceMappingURL=
|
|
39
|
+
//# sourceMappingURL=admin-indexer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"admin-indexer.d.ts","sourceRoot":"","sources":["../../src/ingestion/admin-indexer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAEvD,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAC1C,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,sDAAsD;IACtD,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAoBD,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAoB;IACpC,OAAO,CAAC,kBAAkB,CAAc;IACxC,OAAO,CAAC,kBAAkB,CAAS;gBAEvB,MAAM,EAAE,kBAAkB;IAMtC;;;OAGG;IACG,iBAAiB,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;CA8BhE"}
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* admin Indexer
|
|
3
3
|
*
|
|
4
|
-
* Handles automatic re-indexing of
|
|
5
|
-
* Wire into
|
|
4
|
+
* Handles automatic re-indexing of admin documents when they change.
|
|
5
|
+
* Wire into admin collection afterChange hooks - no admin API calls from here,
|
|
6
6
|
* the event payload carries the document content directly.
|
|
7
7
|
*
|
|
8
|
-
* Usage (in apps/
|
|
9
|
-
* export const
|
|
8
|
+
* Usage (in apps/admin/src/lib/ai/indexer.ts):
|
|
9
|
+
* export const adminIndexer = new AdminIndexer({ db, ingestionPipeline, enabledCollections: ['posts', 'pages'] })
|
|
10
10
|
*
|
|
11
|
-
* In each
|
|
12
|
-
* await
|
|
11
|
+
* In each admin collection afterChange hook:
|
|
12
|
+
* await adminIndexer.onDocumentChanged({ collection: 'posts', id: doc.id, operation, doc })
|
|
13
13
|
*/
|
|
14
14
|
function extractText(doc) {
|
|
15
15
|
// Prefer explicit content/rawContent fields, then JSON fallback
|
|
@@ -32,7 +32,7 @@ function extractTitle(doc) {
|
|
|
32
32
|
return doc.name;
|
|
33
33
|
return undefined;
|
|
34
34
|
}
|
|
35
|
-
export class
|
|
35
|
+
export class AdminIndexer {
|
|
36
36
|
pipeline;
|
|
37
37
|
enabledCollections;
|
|
38
38
|
defaultWorkspaceId;
|
|
@@ -42,7 +42,7 @@ export class CmsIndexer {
|
|
|
42
42
|
this.defaultWorkspaceId = config.defaultWorkspaceId ?? 'default';
|
|
43
43
|
}
|
|
44
44
|
/**
|
|
45
|
-
* Handle
|
|
45
|
+
* Handle an admin document change event.
|
|
46
46
|
* Skips collections not in enabledCollections.
|
|
47
47
|
*/
|
|
48
48
|
async onDocumentChanged(event) {
|
|
@@ -63,7 +63,7 @@ export class CmsIndexer {
|
|
|
63
63
|
const title = extractTitle(event.doc);
|
|
64
64
|
await this.pipeline.ingest({
|
|
65
65
|
workspaceId,
|
|
66
|
-
sourceType: '
|
|
66
|
+
sourceType: 'admin_collection',
|
|
67
67
|
sourceCollection,
|
|
68
68
|
sourceId,
|
|
69
69
|
title,
|
package/dist/ingestion/bm25.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Pure TypeScript BM25 Implementation
|
|
3
3
|
*
|
|
4
|
-
* Built on-demand from top-N vector candidates
|
|
4
|
+
* Built on-demand from top-N vector candidates - no persistent index.
|
|
5
5
|
* k1=1.5, b=0.75 (standard values). No external deps.
|
|
6
6
|
*/
|
|
7
7
|
export interface BM25Document {
|
package/dist/ingestion/bm25.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Pure TypeScript BM25 Implementation
|
|
3
3
|
*
|
|
4
|
-
* Built on-demand from top-N vector candidates
|
|
4
|
+
* Built on-demand from top-N vector candidates - no persistent index.
|
|
5
5
|
* k1=1.5, b=0.75 (standard values). No external deps.
|
|
6
6
|
*/
|
|
7
7
|
const STOP_WORDS = new Set([
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* File Parsers
|
|
3
3
|
*
|
|
4
|
-
* Zero external dependencies
|
|
4
|
+
* Zero external dependencies - pure Node.js + regex/string manipulation.
|
|
5
5
|
* PDF extraction uses a hand-rolled parser that reads text streams directly
|
|
6
6
|
* from the binary PDF format (BT/ET blocks, Tj/TJ operators).
|
|
7
7
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-parsers.d.ts","sourceRoot":"","sources":["../../src/ingestion/file-parsers.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAMD,qBAAa,eAAe;IAC1B,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;CAGlC;AAgBD,qBAAa,cAAc;IACzB,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;
|
|
1
|
+
{"version":3,"file":"file-parsers.d.ts","sourceRoot":"","sources":["../../src/ingestion/file-parsers.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAMD,qBAAa,eAAe;IAC1B,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;CAGlC;AAgBD,qBAAa,cAAc;IACzB,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;CAsClC;AAyBD,qBAAa,UAAU;IACrB,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;CAwBlC;AAmED;;;;;;;;;;;;;GAaG;AACH,qBAAa,SAAS;IACpB,mFAAmF;IACnF,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAIjC,iCAAiC;IACjC,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,GAAG;QAAE,QAAQ,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE;CAqC9F;AAMD,qBAAa,UAAU;IACrB,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;CASlC;AAMD,MAAM,MAAM,MAAM,GAAG,eAAe,GAAG,cAAc,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAE5F,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAgBrD"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* File Parsers
|
|
3
3
|
*
|
|
4
|
-
* Zero external dependencies
|
|
4
|
+
* Zero external dependencies - pure Node.js + regex/string manipulation.
|
|
5
5
|
* PDF extraction uses a hand-rolled parser that reads text streams directly
|
|
6
6
|
* from the binary PDF format (BT/ET blocks, Tj/TJ operators).
|
|
7
7
|
*/
|
|
@@ -31,14 +31,23 @@ export class MarkdownParser {
|
|
|
31
31
|
// Strip frontmatter
|
|
32
32
|
.replace(FRONTMATTER_RE, '')
|
|
33
33
|
// Strip code fences (preserve content inside)
|
|
34
|
-
.replace(CODE_FENCE_RE, (m) =>
|
|
34
|
+
.replace(CODE_FENCE_RE, (m) => {
|
|
35
|
+
// Iteratively strip backtick sequences to handle nested/reconstructed cases
|
|
36
|
+
let result = m;
|
|
37
|
+
let prev;
|
|
38
|
+
do {
|
|
39
|
+
prev = result;
|
|
40
|
+
result = result.replace(/```\w*\n?/g, '');
|
|
41
|
+
} while (result !== prev);
|
|
42
|
+
return result;
|
|
43
|
+
})
|
|
35
44
|
// Strip inline code backticks but keep content
|
|
36
45
|
.replace(INLINE_CODE_RE, (m) => m.slice(1, -1))
|
|
37
46
|
// Strip heading markers
|
|
38
47
|
.replace(HEADING_RE, '')
|
|
39
|
-
// Unwrap bold/italic
|
|
48
|
+
// Unwrap bold/italic - keep inner text
|
|
40
49
|
.replace(EMPHASIS_RE, '$1')
|
|
41
|
-
// Unwrap links
|
|
50
|
+
// Unwrap links - keep link text
|
|
42
51
|
.replace(LINK_RE, '$1')
|
|
43
52
|
// Remove images entirely
|
|
44
53
|
.replace(IMAGE_RE, '')
|
|
@@ -58,8 +67,8 @@ export class MarkdownParser {
|
|
|
58
67
|
// =============================================================================
|
|
59
68
|
// HTML
|
|
60
69
|
// =============================================================================
|
|
61
|
-
const SCRIPT_STYLE_RE = /<(script|style)[^>]*>[
|
|
62
|
-
const HTML_COMMENT_RE = /<!--[
|
|
70
|
+
const SCRIPT_STYLE_RE = /<(script|style)\b[^>]*>(?:[^<]|<(?!\/\1))*<\/\1>/gi;
|
|
71
|
+
const HTML_COMMENT_RE = /<!--(?:[^-]|-(?!->))*-->/g;
|
|
63
72
|
const HTML_ALL_TAGS_RE = /<[^>]+>/g;
|
|
64
73
|
const HTML_ENTITIES = {
|
|
65
74
|
'&': '&',
|
|
@@ -77,9 +86,19 @@ function decodeEntities(text) {
|
|
|
77
86
|
}
|
|
78
87
|
export class HtmlParser {
|
|
79
88
|
parse(input) {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
89
|
+
// Strip script/style tags iteratively to handle nested/malformed cases
|
|
90
|
+
let stripped = input;
|
|
91
|
+
let prev;
|
|
92
|
+
do {
|
|
93
|
+
prev = stripped;
|
|
94
|
+
stripped = stripped.replace(SCRIPT_STYLE_RE, '');
|
|
95
|
+
} while (stripped !== prev);
|
|
96
|
+
// Strip HTML comments iteratively
|
|
97
|
+
do {
|
|
98
|
+
prev = stripped;
|
|
99
|
+
stripped = stripped.replace(HTML_COMMENT_RE, '');
|
|
100
|
+
} while (stripped !== prev);
|
|
101
|
+
const text = decodeEntities(stripped
|
|
83
102
|
.replace(HTML_ALL_TAGS_RE, ' ')
|
|
84
103
|
.replace(/\r\n/g, '\n')
|
|
85
104
|
.replace(/\r/g, '\n')
|
|
@@ -130,20 +149,20 @@ function decodePdfLiteral(raw) {
|
|
|
130
149
|
*/
|
|
131
150
|
function extractTextFromBlock(block) {
|
|
132
151
|
const parts = [];
|
|
133
|
-
// Tj
|
|
134
|
-
const tjRe = /\((
|
|
152
|
+
// Tj - show string: (text) Tj
|
|
153
|
+
const tjRe = /\(((?:[^)\\]|\\.)*)\)\s*Tj/g;
|
|
135
154
|
let m;
|
|
136
155
|
// biome-ignore lint/suspicious/noAssignInExpressions: standard regex loop
|
|
137
156
|
while ((m = tjRe.exec(block)) !== null) {
|
|
138
157
|
if (m[1] !== undefined)
|
|
139
158
|
parts.push(decodePdfLiteral(m[1]));
|
|
140
159
|
}
|
|
141
|
-
// TJ
|
|
160
|
+
// TJ - show array: [(t1)(t2) num …] TJ
|
|
142
161
|
const tjArrayRe = /\[([^\]]*)\]\s*TJ/g;
|
|
143
162
|
// biome-ignore lint/suspicious/noAssignInExpressions: standard regex loop
|
|
144
163
|
while ((m = tjArrayRe.exec(block)) !== null) {
|
|
145
164
|
const inner = m[1] ?? '';
|
|
146
|
-
const strRe = /\((
|
|
165
|
+
const strRe = /\(((?:[^)\\]|\\.)*)\)/g;
|
|
147
166
|
// biome-ignore lint/suspicious/noAssignInExpressions: standard regex loop
|
|
148
167
|
while ((m = strRe.exec(inner)) !== null) {
|
|
149
168
|
if (m[1] !== undefined)
|
|
@@ -151,7 +170,7 @@ function extractTextFromBlock(block) {
|
|
|
151
170
|
}
|
|
152
171
|
}
|
|
153
172
|
// ' and " operators (move-to-next-line-then-show)
|
|
154
|
-
const quoteRe = /\((
|
|
173
|
+
const quoteRe = /\(((?:[^)\\]|\\.)*)\)\s*['"]/g;
|
|
155
174
|
// biome-ignore lint/suspicious/noAssignInExpressions: standard regex loop
|
|
156
175
|
while ((m = quoteRe.exec(block)) !== null) {
|
|
157
176
|
if (m[1] !== undefined)
|
|
@@ -182,7 +201,7 @@ export class PdfParser {
|
|
|
182
201
|
parseBuffer(input) {
|
|
183
202
|
const src = input.toString('binary');
|
|
184
203
|
// Extract all BT…ET text blocks
|
|
185
|
-
const btEtRe = /BT\s([\
|
|
204
|
+
const btEtRe = /BT\s((?:[^E]|E(?!T\b))*)\sET/g;
|
|
186
205
|
const blocks = [];
|
|
187
206
|
let m;
|
|
188
207
|
// biome-ignore lint/suspicious/noAssignInExpressions: standard regex loop
|
|
@@ -200,7 +219,7 @@ export class PdfParser {
|
|
|
200
219
|
const pageCount = countMatch ? Number.parseInt(countMatch[1] ?? '0', 10) : 0;
|
|
201
220
|
// Document title from /Title (string) or /Title <hex>
|
|
202
221
|
let title;
|
|
203
|
-
const titleMatch = /\/Title\s*\((
|
|
222
|
+
const titleMatch = /\/Title\s*\(((?:[^)\\]|\\.)*)\)/.exec(src);
|
|
204
223
|
if (titleMatch) {
|
|
205
224
|
title = decodePdfLiteral(titleMatch[1] ?? '').trim() || undefined;
|
|
206
225
|
}
|
|
@@ -223,7 +242,7 @@ export class JsonParser {
|
|
|
223
242
|
return { text: JSON.stringify(parsed, null, 2) };
|
|
224
243
|
}
|
|
225
244
|
catch {
|
|
226
|
-
// Not valid JSON
|
|
245
|
+
// Not valid JSON - return as-is
|
|
227
246
|
return { text: input.trim() };
|
|
228
247
|
}
|
|
229
248
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Hybrid Search
|
|
2
|
+
* Hybrid Search - Vector + BM25 with Reciprocal Rank Fusion
|
|
3
3
|
*
|
|
4
4
|
* speed mode: cosine vector search only (fast, default)
|
|
5
5
|
* accuracy mode: fetch 50 vector candidates → BM25 on those → RRF fusion → optional LLM rerank
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Hybrid Search
|
|
2
|
+
* Hybrid Search - Vector + BM25 with Reciprocal Rank Fusion
|
|
3
3
|
*
|
|
4
4
|
* speed mode: cosine vector search only (fast, default)
|
|
5
5
|
* accuracy mode: fetch 50 vector candidates → BM25 on those → RRF fusion → optional LLM rerank
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/ingestion/index.ts"],"names":[],"mappings":"AAAA,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/ingestion/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC"}
|
package/dist/ingestion/index.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
import type { Database } from '@revealui/db/client';
|
|
8
8
|
export interface IngestRequest {
|
|
9
9
|
workspaceId: string;
|
|
10
|
-
sourceType: '
|
|
10
|
+
sourceType: 'admin_collection' | 'url' | 'file' | 'text';
|
|
11
11
|
sourceId?: string;
|
|
12
12
|
sourceCollection?: string;
|
|
13
13
|
title?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/ingestion/pipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAOpD,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/ingestion/pipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAOpD,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,kBAAkB,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC;IACzD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,SAAS,GAAG,QAAQ,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAcD,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,EAAE,CAAW;IACrB,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,QAAQ,CAA6B;gBAEjC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;IAOtF,MAAM,CAAC,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IA6EjD,WAAW,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAa3D,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjD,cAAc,CAClB,WAAW,EAAE,MAAM,EACnB,gBAAgB,EAAE,MAAM,EACxB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC;CAiBjB"}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* RAG Vector Service
|
|
3
3
|
*
|
|
4
4
|
* Semantic similarity search over rag_chunks using pgvector cosine distance.
|
|
5
|
-
* Modeled after VectorMemoryService
|
|
5
|
+
* Modeled after VectorMemoryService - same Drizzle cosine-distance pattern.
|
|
6
6
|
*/
|
|
7
7
|
import type { RagChunk, RagDocument } from '@revealui/db/schema/rag';
|
|
8
8
|
export interface RagSearchOptions {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* RAG Vector Service
|
|
3
3
|
*
|
|
4
4
|
* Semantic similarity search over rag_chunks using pgvector cosine distance.
|
|
5
|
-
* Modeled after VectorMemoryService
|
|
5
|
+
* Modeled after VectorMemoryService - same Drizzle cosine-distance pattern.
|
|
6
6
|
*/
|
|
7
7
|
import { getVectorClient } from '@revealui/db/client';
|
|
8
8
|
import { ragChunks, ragDocuments } from '@revealui/db/schema/rag';
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* LLM-based Re-ranker
|
|
3
3
|
*
|
|
4
4
|
* Scores chunks 0–10 for relevance to the query using the configured LLMClient.
|
|
5
|
-
* No cross-encoder model required
|
|
5
|
+
* No cross-encoder model required - uses the existing LLM.
|
|
6
6
|
*/
|
|
7
7
|
import type { LLMClient } from '../llm/client.js';
|
|
8
8
|
import type { RagSearchResult } from './rag-vector-service.js';
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* LLM-based Re-ranker
|
|
3
3
|
*
|
|
4
4
|
* Scores chunks 0–10 for relevance to the query using the configured LLMClient.
|
|
5
|
-
* No cross-encoder model required
|
|
5
|
+
* No cross-encoder model required - uses the existing LLM.
|
|
6
6
|
*/
|
|
7
7
|
const RERANK_SYSTEM_PROMPT = `You are a relevance judge. Given a query and a list of passages, score each passage for relevance to the query on a scale of 0 to 10 (0=not relevant, 10=perfectly relevant). Return ONLY a JSON array of numbers, one score per passage, in the same order as the input. Example: [8, 3, 10, 1]`;
|
|
8
8
|
export async function rerankChunks(query, chunks, llmClient, topK) {
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Splits text into chunks using a hierarchy of separators:
|
|
5
5
|
* double-newline → single-newline → period+space → space
|
|
6
|
-
* Token count is estimated as Math.ceil(length / 4)
|
|
6
|
+
* Token count is estimated as Math.ceil(length / 4) - sufficient for chunking.
|
|
7
7
|
*/
|
|
8
8
|
export interface Chunk {
|
|
9
9
|
content: string;
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Splits text into chunks using a hierarchy of separators:
|
|
5
5
|
* double-newline → single-newline → period+space → space
|
|
6
|
-
* Token count is estimated as Math.ceil(length / 4)
|
|
6
|
+
* Token count is estimated as Math.ceil(length / 4) - sufficient for chunking.
|
|
7
7
|
*/
|
|
8
8
|
const SEPARATORS = ['\n\n', '\n', '. ', ' '];
|
|
9
9
|
function estimateTokens(text) {
|
|
@@ -51,7 +51,7 @@ function recursiveSplit(text, chunkSize, sepIndex = 0) {
|
|
|
51
51
|
return [text];
|
|
52
52
|
const separator = SEPARATORS[sepIndex];
|
|
53
53
|
if (separator === undefined) {
|
|
54
|
-
// No separator left
|
|
54
|
+
// No separator left - hard-split at chunkSize
|
|
55
55
|
const chunks = [];
|
|
56
56
|
for (let i = 0; i < text.length; i += chunkSize) {
|
|
57
57
|
chunks.push(text.slice(i, i + chunkSize));
|
package/dist/llm/client.d.ts
CHANGED
|
@@ -3,24 +3,18 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Single interface for all LLM providers with fallback and rate limiting
|
|
5
5
|
*/
|
|
6
|
-
/**
|
|
7
|
-
* Redact sensitive fields before passing an object to a logger.
|
|
8
|
-
* Replaces API keys, tokens, and authorization headers with `[REDACTED]`.
|
|
9
|
-
* Recurses into nested plain objects; leaves arrays and primitives as-is.
|
|
10
|
-
*/
|
|
11
|
-
export declare function redactSensitiveFields(obj: Record<string, unknown>): Record<string, unknown>;
|
|
12
6
|
import type { Database } from '@revealui/db/client';
|
|
13
7
|
import type { AuditStore } from '../audit/store.js';
|
|
14
8
|
import type { ProviderHealthMonitor } from './provider-health.js';
|
|
15
9
|
import type { Embedding, LLMChatOptions, LLMChunk, LLMEmbedOptions, LLMProvider, LLMResponse, LLMStreamOptions, Message } from './providers/base.js';
|
|
16
10
|
import { type CacheStats, type ResponseCacheOptions } from './response-cache.js';
|
|
17
11
|
import { type SemanticCacheOptions, type SemanticCacheStats } from './semantic-cache.js';
|
|
18
|
-
export type LLMProviderType = 'vultr' | 'groq' | 'ollama' | '
|
|
12
|
+
export type LLMProviderType = 'vultr' | 'groq' | 'ollama' | 'huggingface' | 'inference-snaps';
|
|
19
13
|
export interface LLMClientConfig {
|
|
20
14
|
provider: LLMProviderType;
|
|
21
15
|
apiKey: string;
|
|
22
16
|
/**
|
|
23
|
-
* Dynamic API key resolver
|
|
17
|
+
* Dynamic API key resolver - called before every LLM request.
|
|
24
18
|
* When set, the resolved key replaces `apiKey` on each call.
|
|
25
19
|
* Use this for OAuth tokens or any credential that expires between requests.
|
|
26
20
|
*/
|
|
@@ -30,10 +24,7 @@ export interface LLMClientConfig {
|
|
|
30
24
|
/**
|
|
31
25
|
* Dedicated embedding provider. When set, all embed() calls are routed here
|
|
32
26
|
* instead of the primary provider. Required when the primary provider does not
|
|
33
|
-
* support embeddings
|
|
34
|
-
*
|
|
35
|
-
* Auto-wired by createLLMClientFromEnv() when BITNET_BASE_URL + OLLAMA_BASE_URL
|
|
36
|
-
* are both set.
|
|
27
|
+
* support embeddings natively.
|
|
37
28
|
*/
|
|
38
29
|
embedProvider?: LLMProvider;
|
|
39
30
|
temperature?: number;
|
|
@@ -53,7 +44,7 @@ export interface LLMClientConfig {
|
|
|
53
44
|
enableSemanticCache?: boolean;
|
|
54
45
|
/** Semantic cache options */
|
|
55
46
|
semanticCacheOptions?: SemanticCacheOptions;
|
|
56
|
-
/** Optional health monitor
|
|
47
|
+
/** Optional health monitor - records latency + error rate per provider */
|
|
57
48
|
healthMonitor?: ProviderHealthMonitor;
|
|
58
49
|
}
|
|
59
50
|
export declare class LLMClient {
|
|
@@ -116,13 +107,13 @@ export declare class LLMClient {
|
|
|
116
107
|
* Create an LLM client from environment variables.
|
|
117
108
|
*
|
|
118
109
|
* When LLM_PROVIDER is not set, auto-detects the provider by checking env vars
|
|
119
|
-
* in priority order: INFERENCE_SNAPS →
|
|
110
|
+
* in priority order: INFERENCE_SNAPS → GROQ → OLLAMA.
|
|
120
111
|
*
|
|
121
112
|
* All providers use OpenAI-compatible APIs. No proprietary provider SDKs.
|
|
122
113
|
*
|
|
123
114
|
* Provider defaults:
|
|
124
|
-
* groq →
|
|
125
|
-
* ollama →
|
|
115
|
+
* groq → qwen/qwen3-32b
|
|
116
|
+
* ollama → gemma4:e2b
|
|
126
117
|
*/
|
|
127
118
|
export declare function createLLMClientFromEnv(): LLMClient;
|
|
128
119
|
/**
|
package/dist/llm/client.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/llm/client.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/llm/client.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAIpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,KAAK,EACV,SAAS,EACT,cAAc,EACd,QAAQ,EACR,eAAe,EACf,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,OAAO,EACR,MAAM,qBAAqB,CAAC;AAS7B,OAAO,EAAE,KAAK,UAAU,EAAiB,KAAK,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAChG,OAAO,EAEL,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,MAAM,GAAG,QAAQ,GAAG,aAAa,GAAG,iBAAiB,CAAC;AAE9F,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,eAAe,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,aAAa,CAAC,EAAE,WAAW,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,eAAe,CAAC;IACnC,SAAS,CAAC,EAAE;QACV,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,oFAAoF;IACpF,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,wEAAwE;IACxE,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,6BAA6B;IAC7B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,iEAAiE;IACjE,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,6BAA6B;IAC7B,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,4EAA4E;IAC5E,aAAa,CAAC,EAAE,qBAAqB,CAAC;CACvC;AAQD,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,gBAAgB,CAAC,CAAc;IACvC,OAAO,CAAC,qBAAqB,CAAC,CAAc;IAC5C,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,aAAa,CAAC,CAAgB;IACtC,OAAO,CAAC,aAAa,CAAC,CAAgB;IACtC,OAAO,CAAC,aAAa,CAAC,CAAwB;IAC9C,wFAAwF;IACxF,OAAO,CAAC,aAAa,CAAS;gBAElB,MAAM,EAAE,eAAe;IA8CnC,OAAO,CAAC,cAAc;IAuBtB;;;OAGG;YACW,uBAAuB;IAmBrC,OAAO,CAAC,cAAc;IAoCtB,OAAO,CAAC,aAAa;IAMf,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC;IAmHzE,KAAK,CACT,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EACvB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,SAAS,GAAG,SAAS,EAAE,CAAC;IA2B5B,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,aAAa,CAAC,QAAQ,CAAC;IA0BvF;;;OAGG;IACH,eAAe,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAA;KAAE;IAIlF;;OAEG;IACH,gBAAgB,IAAI,qBAAqB,GAAG,SAAS;IAIrD;;;;OAIG;IACH,qBAAqB,IAAI,UAAU,GAAG,SAAS;IAI/C;;OAEG;IACH,kBAAkB,IAAI,IAAI;IAI1B;;;;OAIG;IACH,qBAAqB,IAAI,kBAAkB,GAAG,SAAS;IAIvD;;OAEG;IACH,kBAAkB,IAAI,IAAI;CAG3B;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,sBAAsB,IAAI,SAAS,CAoElD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,QAAQ,EACZ,UAAU,CAAC,EAAE,UAAU,GACtB,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAiD3B"}
|