@qvac/core 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/compaction/SKILL.md +9 -4
- package/.claude/skills/pr-create/SKILL.md +24 -3
- package/.claude/skills/pr-review/SKILL.md +248 -40
- package/CLAUDE.md +1 -3
- package/README.md +3 -1
- package/dist-lib/client.js +1 -1
- package/dist-lib/index.js +8 -8
- package/dist-lib/lib/api/agents.d.ts +1 -1
- package/dist-lib/lib/api/agents.js +10 -9
- package/dist-lib/lib/api/app-config.js +1 -1
- package/dist-lib/lib/api/approval-rules.js +9 -5
- package/dist-lib/lib/api/attachments.js +25 -3
- package/dist-lib/lib/api/chat-groups.d.ts +1 -1
- package/dist-lib/lib/api/chat-groups.js +5 -4
- package/dist-lib/lib/api/chat.js +8 -8
- package/dist-lib/lib/api/chunks.js +11 -11
- package/dist-lib/lib/api/config.js +1 -1
- package/dist-lib/lib/api/custom-models.js +2 -2
- package/dist-lib/lib/api/devices.d.ts +1 -1
- package/dist-lib/lib/api/devices.js +9 -9
- package/dist-lib/lib/api/diagnostics.js +1 -1
- package/dist-lib/lib/api/effective-ctx-size.js +1 -1
- package/dist-lib/lib/api/execution.js +4 -4
- package/dist-lib/lib/api/files.js +2 -2
- package/dist-lib/lib/api/index.d.ts +1 -0
- package/dist-lib/lib/api/index.js +33 -33
- package/dist-lib/lib/api/instrument.js +1 -1
- package/dist-lib/lib/api/knowledge.js +4 -4
- package/dist-lib/lib/api/logs.js +1 -1
- package/dist-lib/lib/api/members.d.ts +4 -1
- package/dist-lib/lib/api/members.js +88 -18
- package/dist-lib/lib/api/mesh.d.ts +13 -3
- package/dist-lib/lib/api/mesh.js +25 -2
- package/dist-lib/lib/api/model-download.js +1 -1
- package/dist-lib/lib/api/network.js +1 -1
- package/dist-lib/lib/api/notifications.js +3 -3
- package/dist-lib/lib/api/rag.js +7 -7
- package/dist-lib/lib/api/sync.js +1 -1
- package/dist-lib/lib/api/tool-activity.js +4 -4
- package/dist-lib/lib/api/tool-configs.js +5 -5
- package/dist-lib/lib/api/watchable.js +1 -1
- package/dist-lib/lib/assistant/fetch-body.d.ts +5 -0
- package/dist-lib/lib/assistant/fetch-body.js +36 -0
- package/dist-lib/lib/assistant/harness.d.ts +225 -0
- package/dist-lib/lib/assistant/harness.js +154 -0
- package/dist-lib/lib/assistant/history.d.ts +32 -0
- package/dist-lib/lib/assistant/history.js +232 -0
- package/dist-lib/lib/assistant/message.d.ts +45 -0
- package/dist-lib/lib/assistant/message.js +47 -0
- package/dist-lib/lib/assistant/skill-encode.d.ts +21 -0
- package/dist-lib/lib/assistant/skill-encode.js +71 -0
- package/dist-lib/lib/assistant-client.js +2 -2
- package/dist-lib/lib/assistant-tree.js +10 -4
- package/dist-lib/lib/assistant.d.ts +25 -285
- package/dist-lib/lib/assistant.js +250 -637
- package/dist-lib/lib/audio-format.js +2 -2
- package/dist-lib/lib/client.js +2 -2
- package/dist-lib/lib/collabora/convert.js +1 -1
- package/dist-lib/lib/collabora/entry.js +1 -1
- package/dist-lib/lib/collabora/worker.js +1 -1
- package/dist-lib/lib/compaction.d.ts +1 -2
- package/dist-lib/lib/compaction.js +8 -6
- package/dist-lib/lib/constants.js +1 -1
- package/dist-lib/lib/core/account-recovery.d.ts +19 -0
- package/dist-lib/lib/core/account-recovery.js +50 -0
- package/dist-lib/lib/core/agent.js +8 -4
- package/dist-lib/lib/core/app-build.d.ts +1 -1
- package/dist-lib/lib/core/approval-rules.d.ts +1 -0
- package/dist-lib/lib/core/approval-rules.js +3 -1
- package/dist-lib/lib/core/archived-mesh.d.ts +25 -0
- package/dist-lib/lib/core/archived-mesh.js +162 -0
- package/dist-lib/lib/core/chat-activity.js +1 -1
- package/dist-lib/lib/core/chat-group.d.ts +1 -0
- package/dist-lib/lib/core/chat-group.js +24 -4
- package/dist-lib/lib/core/chat-search.js +2 -2
- package/dist-lib/lib/core/chat.js +5 -5
- package/dist-lib/lib/core/chunk.d.ts +1 -1
- package/dist-lib/lib/core/chunk.js +4 -4
- package/dist-lib/lib/core/contract/agents.js +1 -1
- package/dist-lib/lib/core/contract/approval-rules.js +6 -4
- package/dist-lib/lib/core/contract/attestations.js +1 -1
- package/dist-lib/lib/core/contract/chat-group.js +1 -1
- package/dist-lib/lib/core/contract/chat.js +1 -1
- package/dist-lib/lib/core/contract/chunks.js +2 -2
- package/dist-lib/lib/core/contract/compute.d.ts +11 -0
- package/dist-lib/lib/core/contract/compute.js +45 -0
- package/dist-lib/lib/core/contract/devices.js +16 -3
- package/dist-lib/lib/core/contract/index.d.ts +3 -1
- package/dist-lib/lib/core/contract/index.js +19 -16
- package/dist-lib/lib/core/contract/indexing.js +2 -2
- package/dist-lib/lib/core/contract/joined-meshes.js +1 -1
- package/dist-lib/lib/core/contract/knowledge.js +2 -2
- package/dist-lib/lib/core/contract/members.js +2 -2
- package/dist-lib/lib/core/contract/seal.d.ts +2 -0
- package/dist-lib/lib/core/contract/seal.js +8 -0
- package/dist-lib/lib/core/contract/seed-defaults.js +1 -1
- package/dist-lib/lib/core/contract/tool-activity.js +1 -1
- package/dist-lib/lib/core/contract/tool-configs.js +3 -3
- package/dist-lib/lib/core/contract/writers.js +66 -42
- package/dist-lib/lib/core/departures.js +1 -1
- package/dist-lib/lib/core/device-invites.js +1 -1
- package/dist-lib/lib/core/document-attachment.js +2 -2
- package/dist-lib/lib/core/files/attachments.js +3 -3
- package/dist-lib/lib/core/files/index.d.ts +2 -2
- package/dist-lib/lib/core/files/index.js +19 -19
- package/dist-lib/lib/core/files/knowledge.js +3 -3
- package/dist-lib/lib/core/files/preview/image.js +1 -1
- package/dist-lib/lib/core/files/preview/index.js +6 -6
- package/dist-lib/lib/core/files/processing/claims.js +1 -1
- package/dist-lib/lib/core/files/processing/document.js +5 -5
- package/dist-lib/lib/core/files/processing/index.js +8 -8
- package/dist-lib/lib/core/files/processing/ownership.js +1 -1
- package/dist-lib/lib/core/files/processing/protocol.js +3 -3
- package/dist-lib/lib/core/files/processing/source.js +1 -1
- package/dist-lib/lib/core/files/rag/artifact/codec.js +1 -1
- package/dist-lib/lib/core/files/rag/artifact/id.js +1 -1
- package/dist-lib/lib/core/files/rag/artifact/types.d.ts +4 -4
- package/dist-lib/lib/core/files/rag/build.js +2 -2
- package/dist-lib/lib/core/files/rag/chunk/chunking.js +2 -2
- package/dist-lib/lib/core/files/rag/document-types.d.ts +3 -3
- package/dist-lib/lib/core/files/rag/extract/document-types.d.ts +9 -9
- package/dist-lib/lib/core/files/rag/extract/extract.js +7 -7
- package/dist-lib/lib/core/files/rag/extract/html-text.js +1 -1
- package/dist-lib/lib/core/files/rag/extract/office-collabora.js +2 -2
- package/dist-lib/lib/core/files/rag/extract/pdf-platform.js +1 -1
- package/dist-lib/lib/core/files/rag/extract/pdf-text.js +1 -1
- package/dist-lib/lib/core/files/rag/extract/text-decoding.js +1 -1
- package/dist-lib/lib/core/files/rag/extract/types.d.ts +2 -2
- package/dist-lib/lib/core/files/rag/index.js +22 -22
- package/dist-lib/lib/core/files/rag/load.js +5 -5
- package/dist-lib/lib/core/files/rag/policy.js +2 -2
- package/dist-lib/lib/core/files/rag/produce.js +5 -5
- package/dist-lib/lib/core/files/rag/retrieval/coverage.js +2 -2
- package/dist-lib/lib/core/files/rag/retrieval/diversity.js +1 -1
- package/dist-lib/lib/core/files/rag/retrieval/floor.js +2 -2
- package/dist-lib/lib/core/files/rag/retrieval/format.js +1 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +18 -11
- package/dist-lib/lib/core/files/rag/retrieval/knowledge.js +6 -5
- package/dist-lib/lib/core/files/rag/retrieval/scoring.js +2 -2
- package/dist-lib/lib/core/files/rag/retrieval/search.js +7 -4
- package/dist-lib/lib/core/files/rag/retrieval/selection.d.ts +1 -0
- package/dist-lib/lib/core/files/rag/retrieval/selection.js +32 -2
- package/dist-lib/lib/core/files/rag/retrieval/sources.js +2 -2
- package/dist-lib/lib/core/files/rag/retrieval/terms.js +1 -1
- package/dist-lib/lib/core/files/rag/retrieval/types.d.ts +1 -0
- package/dist-lib/lib/core/files/rag/retrieval/web-sources.js +1 -1
- package/dist-lib/lib/core/files/records/chunks.js +1 -1
- package/dist-lib/lib/core/files/records/index.js +8 -8
- package/dist-lib/lib/core/files/records/knowledge.js +2 -2
- package/dist-lib/lib/core/files/records/output.js +1 -1
- package/dist-lib/lib/core/files/records/reclaim.js +5 -5
- package/dist-lib/lib/core/files/records/routing.js +7 -7
- package/dist-lib/lib/core/files/store/blob.js +2 -2
- package/dist-lib/lib/core/files/store/index.js +7 -7
- package/dist-lib/lib/core/files/store/reference.js +2 -2
- package/dist-lib/lib/core/files/store/server.js +1 -1
- package/dist-lib/lib/core/files/store/sweep.js +1 -1
- package/dist-lib/lib/core/files/upload-input.js +2 -2
- package/dist-lib/lib/core/files/upload.d.ts +1 -1
- package/dist-lib/lib/core/files/upload.js +6 -6
- package/dist-lib/lib/core/harness-loan.d.ts +29 -0
- package/dist-lib/lib/core/harness-loan.js +187 -0
- package/dist-lib/lib/core/helpers/index.d.ts +2 -1
- package/dist-lib/lib/core/helpers/index.js +4 -4
- package/dist-lib/lib/core/hosted-agents.js +2 -2
- package/dist-lib/lib/core/identity.d.ts +2 -0
- package/dist-lib/lib/core/identity.js +4 -2
- package/dist-lib/lib/core/invite.d.ts +1 -0
- package/dist-lib/lib/core/invite.js +4 -2
- package/dist-lib/lib/core/local/index.d.ts +12 -3
- package/dist-lib/lib/core/local/index.js +75 -16
- package/dist-lib/lib/core/membership.d.ts +3 -0
- package/dist-lib/lib/core/membership.js +15 -1
- package/dist-lib/lib/core/mesh-seed-handoff.js +1 -1
- package/dist-lib/lib/core/mesh.d.ts +9 -3
- package/dist-lib/lib/core/mesh.js +41 -26
- package/dist-lib/lib/core/migrations/agent-catalog.js +1 -1
- package/dist-lib/lib/core/migrations/index.js +5 -3
- package/dist-lib/lib/core/migrations/membership.js +2 -2
- package/dist-lib/lib/core/migrations/seal.d.ts +3 -0
- package/dist-lib/lib/core/migrations/seal.js +30 -0
- package/dist-lib/lib/core/next-frame.js +1 -1
- package/dist-lib/lib/core/notification.js +1 -1
- package/dist-lib/lib/core/ota-config.js +1 -1
- package/dist-lib/lib/core/secret-handoff.js +1 -1
- package/dist-lib/lib/core/send.js +3 -3
- package/dist-lib/lib/core/skill-overlay-handoff.js +1 -1
- package/dist-lib/lib/core/skill-overlay-receiver.js +2 -2
- package/dist-lib/lib/core/skill-secrets.js +3 -3
- package/dist-lib/lib/core/sync/devices.js +1 -1
- package/dist-lib/lib/core/sync/index.js +8 -5
- package/dist-lib/lib/core/sync-status.js +1 -1
- package/dist-lib/lib/core/system-info.js +1 -1
- package/dist-lib/lib/core/tool-activity.js +2 -2
- package/dist-lib/lib/core/tool-configs.js +2 -2
- package/dist-lib/lib/core.d.ts +41 -3
- package/dist-lib/lib/core.js +525 -90
- package/dist-lib/lib/credential-pusher.js +4 -4
- package/dist-lib/lib/deadline.js +1 -1
- package/dist-lib/lib/fs-drive-bridge.js +4 -2
- package/dist-lib/lib/integrations/index.js +3 -3
- package/dist-lib/lib/integrations/oauth-providers.js +1 -1
- package/dist-lib/lib/integrations/setup-routes.d.ts +5 -5
- package/dist-lib/lib/integrations/setup-routes.js +3 -3
- package/dist-lib/lib/knowledge-search.js +1 -1
- package/dist-lib/lib/log.js +3 -3
- package/dist-lib/lib/mesh-scoped-engine.d.ts +3 -0
- package/dist-lib/lib/mesh-scoped-engine.js +96 -0
- package/dist-lib/lib/model-context.d.ts +2 -2
- package/dist-lib/lib/model-context.js +1 -1
- package/dist-lib/lib/model-download.js +6 -6
- package/dist-lib/lib/oauth/asana.js +3 -3
- package/dist-lib/lib/oauth/browser-connect.js +1 -1
- package/dist-lib/lib/oauth/client.js +2 -2
- package/dist-lib/lib/oauth/common.d.ts +1 -1
- package/dist-lib/lib/oauth/common.js +1 -1
- package/dist-lib/lib/oauth/flow-core.js +3 -3
- package/dist-lib/lib/oauth/flow.js +2 -2
- package/dist-lib/lib/oauth/google.js +4 -4
- package/dist-lib/lib/oauth/host.js +3 -3
- package/dist-lib/lib/oauth/loopback.js +1 -1
- package/dist-lib/lib/oauth/notion.js +3 -3
- package/dist-lib/lib/oauth/providers.js +1 -1
- package/dist-lib/lib/oauth/resolve.js +1 -1
- package/dist-lib/lib/oauth/spotify.js +3 -3
- package/dist-lib/lib/oauth/token-exchange.js +1 -1
- package/dist-lib/lib/oauth.client.js +2 -2
- package/dist-lib/lib/oauth.js +12 -12
- package/dist-lib/lib/run-notices.d.ts +7 -7
- package/dist-lib/lib/run-transcript.js +1 -1
- package/dist-lib/schema/build.js +12 -6
- package/dist-lib/schema/content.d.ts +11 -11
- package/dist-lib/schema/engine.js +193 -30
- package/dist-lib/schema/field-order.js +2 -2
- package/dist-lib/schema/local.js +18 -4
- package/dist-lib/schema/qvac.js +15 -6
- package/dist-lib/spec/bind.js +41 -0
- package/dist-lib/spec/calls.js +12 -0
- package/dist-lib/spec/capabilities.d.ts +36 -12
- package/dist-lib/spec/hrpc/hrpc.json +161 -5
- package/dist-lib/spec/hrpc/index.js +154 -9
- package/dist-lib/spec/hrpc/messages.js +549 -129
- package/dist-lib/spec/hyperschema/index.js +549 -129
- package/dist-lib/spec/hyperschema/schema.json +370 -40
- package/dist-lib/spec/hyperschema/types.d.ts +85 -1
- package/dist-lib/spec/local/hyperdb/db.json +15 -0
- package/dist-lib/spec/local/hyperdb/index.js +70 -1
- package/dist-lib/spec/local/hyperdb/messages.js +300 -117
- package/dist-lib/spec/local/hyperschema/index.js +97 -4
- package/dist-lib/spec/local/hyperschema/schema.json +58 -1
- package/dist-lib/spec/local/hyperschema/types.d.ts +14 -0
- package/dist-lib/spec/mesh/hyperdb/messages.js +393 -288
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +6 -0
- package/dist-lib/spec/mesh/hyperdispatch/index.js +18 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +176 -85
- package/dist-lib/spec/mesh/hyperschema/index.js +176 -85
- package/dist-lib/spec/mesh/hyperschema/schema.json +71 -21
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +14 -1
- package/package.json +16 -10
- package/tui/README.md +7 -0
- package/tui/app.mjs +311 -14
- package/tui/lib/busy-words.mjs +57 -5
- package/tui/lib/export.mjs +1 -0
- package/tui/lib/render.mjs +9 -2
- package/tui/main.mjs +5 -2
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
export declare const RAG_DOCUMENT_KIND: {
|
|
2
|
-
readonly pdf:
|
|
3
|
-
readonly office:
|
|
4
|
-
readonly rtf:
|
|
5
|
-
readonly html:
|
|
6
|
-
readonly csv:
|
|
7
|
-
readonly tsv:
|
|
8
|
-
readonly text:
|
|
9
|
-
readonly unsupported:
|
|
2
|
+
readonly pdf: 'pdf';
|
|
3
|
+
readonly office: 'office';
|
|
4
|
+
readonly rtf: 'rtf';
|
|
5
|
+
readonly html: 'html';
|
|
6
|
+
readonly csv: 'csv';
|
|
7
|
+
readonly tsv: 'tsv';
|
|
8
|
+
readonly text: 'text';
|
|
9
|
+
readonly unsupported: 'unsupported';
|
|
10
10
|
};
|
|
11
11
|
export type RagDocumentKind = (typeof RAG_DOCUMENT_KIND)[keyof typeof RAG_DOCUMENT_KIND];
|
|
12
|
-
export declare const RAG_ACCEPT_EXTENSIONS: readonly [
|
|
12
|
+
export declare const RAG_ACCEPT_EXTENSIONS: readonly ['.pdf', '.docx', '.doc', '.docm', '.dot', '.dotx', '.odt', '.fodt', '.ott', '.txt', '.md', '.markdown', '.rtf', '.html', '.htm', '.xhtml', '.log', '.json', '.jsonl', '.csv', '.tsv', '.xml', '.yaml', '.yml'];
|
|
13
13
|
export declare function classifyRagDocument(input: {
|
|
14
14
|
readonly fileName?: string;
|
|
15
15
|
readonly mimeType?: string;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { extractCsvText } from
|
|
2
|
-
import { classifyRagDocument, RAG_DOCUMENT_KIND } from
|
|
3
|
-
import { extractHtmlText } from
|
|
4
|
-
import { extractPdfText } from
|
|
5
|
-
import { extractRtfText } from
|
|
6
|
-
import { decodeTextBytes } from
|
|
7
|
-
import { RAG_EXTRACT_ERROR_CODE, RagExtractError } from
|
|
1
|
+
import { extractCsvText } from './csv-text.js';
|
|
2
|
+
import { classifyRagDocument, RAG_DOCUMENT_KIND } from './document-types.js';
|
|
3
|
+
import { extractHtmlText } from './html-text.js';
|
|
4
|
+
import { extractPdfText } from './pdf-text.js';
|
|
5
|
+
import { extractRtfText } from './rtf-text.js';
|
|
6
|
+
import { decodeTextBytes } from './text-decoding.js';
|
|
7
|
+
import { RAG_EXTRACT_ERROR_CODE, RagExtractError } from './types.js';
|
|
8
8
|
export async function extractRagDocumentFromBytes(input, opts = {}) {
|
|
9
9
|
const fileName = input.fileName ?? input.title ?? input.sourcePath ?? 'document';
|
|
10
10
|
const kind = classifyRagDocument({
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { formatHtmlTables } from
|
|
1
|
+
import { formatHtmlTables } from './html-table-text.js';
|
|
2
2
|
const DROP_BLOCKS = /<(script|style|noscript|template|svg|head)\b[^>]*>[\s\S]*?<\/\1\s*>/gi;
|
|
3
3
|
const COMMENT = /<!--[\s\S]*?-->/g;
|
|
4
4
|
const HEADING_OPEN = /<h([1-6])\b[^>]*>/gi;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { convertToText } from
|
|
2
|
-
import { MAX_OFFICE_BYTES } from
|
|
1
|
+
import { convertToText } from '../../../../collabora/convert.js';
|
|
2
|
+
import { MAX_OFFICE_BYTES } from '../limits.js';
|
|
3
3
|
export const extractOfficeText = (bytes, fileName, opts) => {
|
|
4
4
|
if (bytes.byteLength > MAX_OFFICE_BYTES) {
|
|
5
5
|
throw new Error(`document is too large to convert: ${bytes.byteLength} bytes, limit ${MAX_OFFICE_BYTES}`);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
2
|
import { AbortController, AbortSignal } from 'bare-abort-controller';
|
|
3
3
|
import { ReadableStream } from 'web-streams-polyfill/dist/ponyfill.mjs';
|
|
4
|
-
import { decodeLatin1, decodeUtf16 } from
|
|
4
|
+
import { decodeLatin1, decodeUtf16 } from './utf.js';
|
|
5
5
|
const g = globalThis;
|
|
6
6
|
if (!g.ReadableStream)
|
|
7
7
|
g.ReadableStream = ReadableStream;
|
|
@@ -26,8 +26,8 @@ export type RagFileInput = {
|
|
|
26
26
|
readonly chatId?: string;
|
|
27
27
|
};
|
|
28
28
|
export declare const RAG_EXTRACT_ERROR_CODE: {
|
|
29
|
-
readonly unsupportedFile:
|
|
30
|
-
readonly extractionFailed:
|
|
29
|
+
readonly unsupportedFile: 'RAG_UNSUPPORTED_FILE';
|
|
30
|
+
readonly extractionFailed: 'RAG_EXTRACTION_FAILED';
|
|
31
31
|
};
|
|
32
32
|
export type RagExtractErrorCode = (typeof RAG_EXTRACT_ERROR_CODE)[keyof typeof RAG_EXTRACT_ERROR_CODE];
|
|
33
33
|
export declare class RagExtractError extends Error {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import ScopeLock from 'scope-lock';
|
|
2
|
-
import { pipelineVersionFor } from
|
|
2
|
+
import { pipelineVersionFor } from './artifact/id.js';
|
|
3
3
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
4
|
-
import { IndexingPass } from
|
|
5
|
-
import { HydratePass } from
|
|
6
|
-
import { canIndexChat } from
|
|
4
|
+
import { IndexingPass } from './produce.js';
|
|
5
|
+
import { HydratePass } from './load.js';
|
|
6
|
+
import { canIndexChat } from './policy.js';
|
|
7
7
|
export class RagProcessor {
|
|
8
8
|
id = 'rag';
|
|
9
9
|
input = 'knowledge';
|
|
@@ -59,21 +59,21 @@ export class RagProcessor {
|
|
|
59
59
|
}
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
|
-
export { encodeRagArtifact } from
|
|
63
|
-
export { isSupportedRagDocument } from
|
|
64
|
-
export { MAX_KNOWLEDGE_BYTES } from
|
|
65
|
-
export { canIndexChat } from
|
|
66
|
-
export { citedByPhraseOverlap } from
|
|
67
|
-
export { formatRagContext } from
|
|
68
|
-
export { formatFullDocumentContext, isFullTextHit, isTruncatedFullTextHit } from
|
|
69
|
-
export { RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, chooseRagNotice } from
|
|
70
|
-
export { createRagSourceTagFilter } from
|
|
71
|
-
export { sourcesFromHits } from
|
|
72
|
-
export { WEB_TOOLS, webSourcesFromToolResult, withoutDataChannel } from
|
|
73
|
-
export { MemoryVectorIndex } from
|
|
74
|
-
export { fullTextBudgetChars, fullTextHits } from
|
|
75
|
-
export { retrieveRagKnowledge } from
|
|
76
|
-
export { searchRagChunks } from
|
|
77
|
-
export { referencesPendingFile } from
|
|
78
|
-
export { isWholeDocumentQuery } from
|
|
79
|
-
export { referencedDocuments } from
|
|
62
|
+
export { encodeRagArtifact } from './artifact/codec.js';
|
|
63
|
+
export { isSupportedRagDocument } from './extract/document-types.js';
|
|
64
|
+
export { MAX_KNOWLEDGE_BYTES } from './limits.js';
|
|
65
|
+
export { canIndexChat } from './policy.js';
|
|
66
|
+
export { citedByPhraseOverlap } from './retrieval/citation-match.js';
|
|
67
|
+
export { formatRagContext } from './retrieval/format.js';
|
|
68
|
+
export { formatFullDocumentContext, isFullTextHit, isTruncatedFullTextHit } from './retrieval/full-text.js';
|
|
69
|
+
export { RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, chooseRagNotice } from './retrieval/notices.js';
|
|
70
|
+
export { createRagSourceTagFilter } from './retrieval/source-tag-filter.js';
|
|
71
|
+
export { sourcesFromHits } from './retrieval/sources.js';
|
|
72
|
+
export { WEB_TOOLS, webSourcesFromToolResult, withoutDataChannel } from './retrieval/web-sources.js';
|
|
73
|
+
export { MemoryVectorIndex } from './vector/memory.js';
|
|
74
|
+
export { fullTextBudgetChars, fullTextHits } from './retrieval/full-text.js';
|
|
75
|
+
export { retrieveRagKnowledge } from './retrieval/knowledge.js';
|
|
76
|
+
export { searchRagChunks } from './retrieval/search.js';
|
|
77
|
+
export { referencesPendingFile } from './retrieval/target.js';
|
|
78
|
+
export { isWholeDocumentQuery } from './retrieval/whole-document-query.js';
|
|
79
|
+
export { referencedDocuments } from './retrieval/target.js';
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { decodeRagArtifact } from
|
|
2
|
-
import { pipelineVersionFor, contentHash } from
|
|
3
|
-
import { compareBlobReferences } from
|
|
4
|
-
import { buildArtifactPayload, EmbedderFailure } from
|
|
5
|
-
import { EMBED_RETRY_LIMIT } from
|
|
1
|
+
import { decodeRagArtifact } from './artifact/codec.js';
|
|
2
|
+
import { pipelineVersionFor, contentHash } from './artifact/id.js';
|
|
3
|
+
import { compareBlobReferences } from '../store/index.js';
|
|
4
|
+
import { buildArtifactPayload, EmbedderFailure } from './build.js';
|
|
5
|
+
import { EMBED_RETRY_LIMIT } from './produce.js';
|
|
6
6
|
export function pickArtifact(rows, opts) {
|
|
7
7
|
const candidates = rows.filter((row) => !opts.skip.has(row.id) &&
|
|
8
8
|
!sameReference(opts.failed(row.id), row.artifactBlobId, opts.localCoreKey) &&
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { toolsetEnabled } from '@qvac/harness/lib/tools/toolsets.ts';
|
|
2
|
-
import { getAgent, isOrphaned } from
|
|
3
|
-
import { getChat } from
|
|
2
|
+
import { getAgent, isOrphaned } from '../../agent.js';
|
|
3
|
+
import { getChat } from '../../chat.js';
|
|
4
4
|
export async function canIndexChat(mesh, deviceId, chatId) {
|
|
5
5
|
if (!deviceId) {
|
|
6
6
|
return false;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { encodeRagArtifact } from
|
|
2
|
-
import { contentHash, documentIndexArtifactId, pipelineVersionFor } from
|
|
3
|
-
import { classifyRagDocument, RAG_DOCUMENT_KIND } from
|
|
4
|
-
import { buildArtifactPayload, EmbedderFailure } from
|
|
5
|
-
import { isClaimLive } from
|
|
1
|
+
import { encodeRagArtifact } from './artifact/codec.js';
|
|
2
|
+
import { contentHash, documentIndexArtifactId, pipelineVersionFor } from './artifact/id.js';
|
|
3
|
+
import { classifyRagDocument, RAG_DOCUMENT_KIND } from './extract/document-types.js';
|
|
4
|
+
import { buildArtifactPayload, EmbedderFailure } from './build.js';
|
|
5
|
+
import { isClaimLive } from '../processing/index.js';
|
|
6
6
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
7
7
|
const UNSUPPORTED = 'unsupported';
|
|
8
8
|
const ERROR_LIMIT = 300;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { parseRagMetadata } from
|
|
2
|
-
import { estimateRagTokens } from
|
|
1
|
+
import { parseRagMetadata } from '../chunk/metadata.js';
|
|
2
|
+
import { estimateRagTokens } from '../chunk/tokenizer.js';
|
|
3
3
|
const DEFAULT_COVERAGE_BUDGET_TOKENS = 3072;
|
|
4
4
|
export function selectCoverage(hits, { budgetTokens = DEFAULT_COVERAGE_BUDGET_TOKENS } = {}) {
|
|
5
5
|
const documents = groupByDocument(hits);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { hitMatchesFilters, parseRagMetadata } from
|
|
1
|
+
import { hitMatchesFilters, parseRagMetadata } from '../chunk/metadata.js';
|
|
2
2
|
export function scopeFilterFor(filters) {
|
|
3
3
|
const liveSourceIds = filters.liveSourceIds ? new Set(filters.liveSourceIds) : undefined;
|
|
4
4
|
if (!filters.chatId &&
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import
|
|
2
|
-
import { parseRagMetadata } from
|
|
1
|
+
import '../../../../logger.js';
|
|
2
|
+
import { parseRagMetadata } from '../chunk/metadata.js';
|
|
3
3
|
export const RAG_MIN_SIMILARITY = 0.4;
|
|
4
4
|
export function applySimilarityFloor(hits, targetDocumentId) {
|
|
5
5
|
if (targetDocumentId)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { parseRagMetadata } from
|
|
1
|
+
import { parseRagMetadata } from '../chunk/metadata.js';
|
|
2
2
|
export function formatRagContext(hits, availableFiles = []) {
|
|
3
3
|
const manifest = availableFiles.length > 0 ? [`Available files in this chat: ${availableFiles.join(', ')}`] : [];
|
|
4
4
|
return [
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import { parseRagMetadata } from
|
|
2
|
-
import { classifyRagDocument, RAG_DOCUMENT_KIND } from
|
|
3
|
-
import { extractRagDocumentFromBytes } from
|
|
4
|
-
import { extractPdfHead } from
|
|
1
|
+
import { parseRagMetadata } from '../chunk/metadata.js';
|
|
2
|
+
import { classifyRagDocument, RAG_DOCUMENT_KIND } from '../extract/document-types.js';
|
|
3
|
+
import { extractRagDocumentFromBytes } from '../extract/extract.js';
|
|
4
|
+
import { extractPdfHead } from '../extract/pdf-text.js';
|
|
5
5
|
const DEFAULT_FULL_TEXT_BUDGET_TOKENS = 4096;
|
|
6
6
|
const CHARS_PER_TOKEN = 4;
|
|
7
|
-
const FULL_TEXT_MAX_BYTES = 20 * 1024 * 1024;
|
|
8
7
|
const FULL_TEXT_KINDS = new Set([
|
|
9
8
|
RAG_DOCUMENT_KIND.pdf,
|
|
10
9
|
RAG_DOCUMENT_KIND.rtf,
|
|
@@ -71,15 +70,13 @@ export async function fullTextHits(mesh, rows, budgetChars, { head = false } = {
|
|
|
71
70
|
const hits = [];
|
|
72
71
|
let used = 0;
|
|
73
72
|
for (const [index, row] of rows.entries()) {
|
|
74
|
-
if (row.byteLength > FULL_TEXT_MAX_BYTES)
|
|
75
|
-
return null;
|
|
76
73
|
const remaining = budgetChars - used;
|
|
77
74
|
if (!head && kinds[index] !== RAG_DOCUMENT_KIND.pdf && row.byteLength > remaining)
|
|
78
75
|
return null;
|
|
79
76
|
let read;
|
|
80
77
|
try {
|
|
81
78
|
const bytes = await mesh.files.readBytes(row.blobId);
|
|
82
|
-
read = await extractFullText(kinds[index], bytes, row, remaining);
|
|
79
|
+
read = await extractFullText(kinds[index], bytes, row, remaining, head);
|
|
83
80
|
}
|
|
84
81
|
catch (err) {
|
|
85
82
|
logger.warn('rag', 'full-text extraction failed; falling back to retrieval', { err });
|
|
@@ -119,7 +116,8 @@ function coverageOf(read, included) {
|
|
|
119
116
|
}
|
|
120
117
|
return { unit: 'chars', included, total: read.totalChars ?? read.text.length };
|
|
121
118
|
}
|
|
122
|
-
|
|
119
|
+
const MAX_BYTES_PER_CHAR = 4;
|
|
120
|
+
async function extractFullText(kind, bytes, row, remainingChars, head) {
|
|
123
121
|
if (kind === RAG_DOCUMENT_KIND.pdf) {
|
|
124
122
|
const pdf = await extractPdfHead(bytes, { maxChars: remainingChars });
|
|
125
123
|
return {
|
|
@@ -128,10 +126,19 @@ async function extractFullText(kind, bytes, row, remainingChars) {
|
|
|
128
126
|
pages: { read: pdf.pagesRead, total: pdf.totalPages }
|
|
129
127
|
};
|
|
130
128
|
}
|
|
129
|
+
const maxPrefixBytes = remainingChars * MAX_BYTES_PER_CHAR;
|
|
130
|
+
const prefix = head && bytes.length > maxPrefixBytes;
|
|
131
131
|
const document = await extractRagDocumentFromBytes({
|
|
132
132
|
fileName: row.fileName,
|
|
133
133
|
mimeType: row.mimeType,
|
|
134
|
-
bytes
|
|
134
|
+
bytes: prefix ? bytes.subarray(0, maxPrefixBytes) : bytes
|
|
135
135
|
});
|
|
136
|
-
|
|
136
|
+
if (!prefix) {
|
|
137
|
+
return { text: document.content, complete: true, totalChars: document.content.length };
|
|
138
|
+
}
|
|
139
|
+
return {
|
|
140
|
+
text: document.content.replace(/\uFFFD$/, ''),
|
|
141
|
+
complete: false,
|
|
142
|
+
totalChars: bytes.length
|
|
143
|
+
};
|
|
137
144
|
}
|
|
@@ -1,14 +1,15 @@
|
|
|
1
|
-
import { applySimilarityFloor, logSimilarityScores } from
|
|
2
|
-
import { searchRagChunks } from
|
|
3
|
-
import { DEFAULT_CHUNK_BUDGET_TOKENS, MAX_SELECTED_CHUNKS } from
|
|
4
|
-
import { resolveTargetDocument } from
|
|
5
|
-
import { isWholeDocumentQuery } from
|
|
1
|
+
import { applySimilarityFloor, logSimilarityScores } from './floor.js';
|
|
2
|
+
import { searchRagChunks } from './search.js';
|
|
3
|
+
import { DEFAULT_CHUNK_BUDGET_TOKENS, MAX_SELECTED_CHUNKS } from './selection.js';
|
|
4
|
+
import { resolveTargetDocument } from './target.js';
|
|
5
|
+
import { isWholeDocumentQuery } from './whole-document-query.js';
|
|
6
6
|
export async function retrieveRagKnowledge(query, deps) {
|
|
7
7
|
if (query.scopeDocumentIds?.length) {
|
|
8
8
|
const hits = await searchRagChunks({
|
|
9
9
|
query: query.query,
|
|
10
10
|
chatId: query.chatId,
|
|
11
11
|
liveSourceIds: query.liveSourceIds,
|
|
12
|
+
coverDocuments: query.scopeDocumentIds.length > 1,
|
|
12
13
|
...scopedShape(query.query, query.scopeDocumentIds)
|
|
13
14
|
}, deps);
|
|
14
15
|
logSimilarityScores(hits, hits);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { normalize } from 'asenion';
|
|
2
|
-
import { parseRagMetadata } from
|
|
3
|
-
import { extractSalientTerms, isMeaningfulToken } from
|
|
2
|
+
import { parseRagMetadata } from '../chunk/metadata.js';
|
|
3
|
+
import { extractSalientTerms, isMeaningfulToken } from './terms.js';
|
|
4
4
|
const SEM_MAX_WEIGHT = 0.7;
|
|
5
5
|
const SEM_MEAN_WEIGHT = 0.3;
|
|
6
6
|
const SEM_TOP_K = 3;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { selectCoverage } from
|
|
2
|
-
import { applyPerSourceCap, scopeFilterFor } from
|
|
3
|
-
import { scoreDocuments } from
|
|
4
|
-
import { selectChunks } from
|
|
1
|
+
import { selectCoverage } from './coverage.js';
|
|
2
|
+
import { applyPerSourceCap, scopeFilterFor } from './diversity.js';
|
|
3
|
+
import { scoreDocuments } from './scoring.js';
|
|
4
|
+
import { selectChunks, selectScopedChunks } from './selection.js';
|
|
5
5
|
const DEFAULT_LIMIT = 5;
|
|
6
6
|
export async function searchRagChunks(query, deps) {
|
|
7
7
|
const trimmed = query.query.trim();
|
|
@@ -21,6 +21,9 @@ export async function searchRagChunks(query, deps) {
|
|
|
21
21
|
: {})
|
|
22
22
|
});
|
|
23
23
|
}
|
|
24
|
+
if (query.coverDocuments) {
|
|
25
|
+
return selectScopedChunks(scoreDocuments(hits, query.query), query.chunkBudgetTokens);
|
|
26
|
+
}
|
|
24
27
|
if (useDocumentFirst(query))
|
|
25
28
|
return documentFirstSelect(query, hits);
|
|
26
29
|
return applyPerSourceCap(hits, query.limit ?? DEFAULT_LIMIT);
|
|
@@ -8,3 +8,4 @@ export type SelectChunksOptions = {
|
|
|
8
8
|
readonly perDocChunkCap?: number;
|
|
9
9
|
};
|
|
10
10
|
export declare function selectChunks(ranked: ReadonlyArray<ScoredDocument>, options?: SelectChunksOptions): ReadonlyArray<VectorHit>;
|
|
11
|
+
export declare function selectScopedChunks(ranked: ReadonlyArray<ScoredDocument>, budgetTokens?: number): ReadonlyArray<VectorHit>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { parseRagMetadata } from
|
|
2
|
-
import { estimateRagTokens } from
|
|
1
|
+
import { parseRagMetadata } from '../chunk/metadata.js';
|
|
2
|
+
import { estimateRagTokens, tokenizeForRag } from '../chunk/tokenizer.js';
|
|
3
3
|
const DEFAULT_MAX_DOCS = 8;
|
|
4
4
|
const DEFAULT_PER_DOC_CHUNK_CAP = 4;
|
|
5
5
|
export const DEFAULT_CHUNK_BUDGET_TOKENS = 3072;
|
|
@@ -23,6 +23,36 @@ export function selectChunks(ranked, options = {}) {
|
|
|
23
23
|
.sort((left, right) => left.docRank - right.docRank || left.chunkIndex - right.chunkIndex)
|
|
24
24
|
.map((entry) => entry.hit);
|
|
25
25
|
}
|
|
26
|
+
export function selectScopedChunks(ranked, budgetTokens = DEFAULT_CHUNK_BUDGET_TOKENS) {
|
|
27
|
+
const share = Math.floor(budgetTokens / ranked.length);
|
|
28
|
+
const selected = [];
|
|
29
|
+
for (const document of ranked) {
|
|
30
|
+
let remaining = share;
|
|
31
|
+
let count = 0;
|
|
32
|
+
for (const hit of document.hits) {
|
|
33
|
+
if (remaining === 0 || count === DEFAULT_PER_DOC_CHUNK_CAP)
|
|
34
|
+
break;
|
|
35
|
+
const tokens = tokenizeForRag(hit.content);
|
|
36
|
+
if (tokens.length > remaining) {
|
|
37
|
+
if (count > 0)
|
|
38
|
+
break;
|
|
39
|
+
selected.push({
|
|
40
|
+
...hit,
|
|
41
|
+
content: hit.content.slice(0, tokens[remaining - 1].end),
|
|
42
|
+
metadataJson: JSON.stringify({
|
|
43
|
+
...parseRagMetadata(hit.metadataJson),
|
|
44
|
+
tokenCount: remaining
|
|
45
|
+
})
|
|
46
|
+
});
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
selected.push(hit);
|
|
50
|
+
remaining -= tokens.length;
|
|
51
|
+
count++;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return selected;
|
|
55
|
+
}
|
|
26
56
|
function seedCoverage(state) {
|
|
27
57
|
state.topDocs.forEach((doc, rank) => {
|
|
28
58
|
const best = doc.hits[0];
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { MAX_RAG_SOURCES_PER_CHUNK, RAG_SOURCE_EXCERPT_CHARS } from
|
|
2
|
-
import { parseRagMetadata } from
|
|
1
|
+
import { MAX_RAG_SOURCES_PER_CHUNK, RAG_SOURCE_EXCERPT_CHARS } from '../../../../../schema/qvac.js';
|
|
2
|
+
import { parseRagMetadata } from '../chunk/metadata.js';
|
|
3
3
|
export { RAG_SOURCE_EXCERPT_CHARS };
|
|
4
4
|
export function sourcesFromHits(hits) {
|
|
5
5
|
return hits.slice(0, MAX_RAG_SOURCES_PER_CHUNK).map((hit, index) => {
|
|
@@ -10,6 +10,7 @@ export type RagRetrievalQuery = RagScopeFilters & {
|
|
|
10
10
|
readonly query: string;
|
|
11
11
|
readonly limit?: number;
|
|
12
12
|
readonly documentFirst?: boolean;
|
|
13
|
+
readonly coverDocuments?: boolean;
|
|
13
14
|
readonly chunkBudgetTokens?: number;
|
|
14
15
|
readonly coverage?: boolean;
|
|
15
16
|
readonly perDocChunkCap?: number;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { MAX_RAG_SOURCES_PER_CHUNK, MAX_RAG_SOURCE_URL_CHARS, RAG_SOURCE_EXCERPT_CHARS } from
|
|
1
|
+
import { MAX_RAG_SOURCES_PER_CHUNK, MAX_RAG_SOURCE_URL_CHARS, RAG_SOURCE_EXCERPT_CHARS } from '../../../../../schema/qvac.js';
|
|
2
2
|
export const WEB_TOOLS = ['web_search', 'web_fetch'];
|
|
3
3
|
export function webSourcesFromToolResult(result) {
|
|
4
4
|
const results = structuredResults(result);
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
export { addKnowledge, addKnowledgeArtifact, assertRemovableAt, findKnowledgeArtifactsByKnowledge, getKnowledge, getKnowledgeArtifact, listKnowledge, listKnowledgeArtifactsByKnowledge, removeKnowledge } from
|
|
2
|
-
export { attachTurnDocumentIds, hasLiveSource, isKnowledgeRoutedAttachment, routeAttachmentToKnowledge } from
|
|
3
|
-
export { groupAttachmentsByMessage } from
|
|
4
|
-
export { syncAttachmentChunk } from
|
|
5
|
-
export { sweepTombstonedBlobs, syncArtifactArrival, syncKnowledgeTombstone } from
|
|
6
|
-
export { referencedBlobs } from
|
|
7
|
-
export { assertArtifactOutput, isProcessorOutput, processorArtifactId, isProcessorArtifactId } from
|
|
8
|
-
export { attachmentRecord, fileRecord, fileIds, scopedFile, claimSource } from
|
|
1
|
+
export { addKnowledge, addKnowledgeArtifact, assertRemovableAt, findKnowledgeArtifactsByKnowledge, getKnowledge, getKnowledgeArtifact, listKnowledge, listKnowledgeArtifactsByKnowledge, removeKnowledge } from './knowledge.js';
|
|
2
|
+
export { attachTurnDocumentIds, hasLiveSource, isKnowledgeRoutedAttachment, routeAttachmentToKnowledge } from './routing.js';
|
|
3
|
+
export { groupAttachmentsByMessage } from './linkage.js';
|
|
4
|
+
export { syncAttachmentChunk } from './chunks.js';
|
|
5
|
+
export { sweepTombstonedBlobs, syncArtifactArrival, syncKnowledgeTombstone } from './reclaim.js';
|
|
6
|
+
export { referencedBlobs } from './sweep.js';
|
|
7
|
+
export { assertArtifactOutput, isProcessorOutput, processorArtifactId, isProcessorArtifactId } from './output.js';
|
|
8
|
+
export { attachmentRecord, fileRecord, fileIds, scopedFile, claimSource } from './source.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { randomId } from
|
|
2
|
-
import { assertArtifactOutput } from
|
|
1
|
+
import { randomId } from '../../id.js';
|
|
2
|
+
import { assertArtifactOutput } from './output.js';
|
|
3
3
|
const KNOWLEDGE = '@qvac/knowledge';
|
|
4
4
|
const ACTIVE_KNOWLEDGE_BY_CHAT = '@qvac/active-knowledge-by-chat';
|
|
5
5
|
const KNOWLEDGE_ARTIFACTS = '@qvac/knowledge-artifacts';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { MAX_REFERENCE_CHARS, parseBlobReference } from
|
|
1
|
+
import { MAX_REFERENCE_CHARS, parseBlobReference } from '../store/index.js';
|
|
2
2
|
const PROCESSOR_ID_SEPARATOR = ':';
|
|
3
3
|
export function processorArtifactId({ processorId, hash }) {
|
|
4
4
|
return `${processorId}${PROCESSOR_ID_SEPARATOR}${hash}`;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { hasLiveSource } from
|
|
2
|
-
import { removeChatAttachments, removeChatKnowledge } from
|
|
3
|
-
import { listChunks } from
|
|
4
|
-
import { getKnowledge, listKnowledgeArtifactsByKnowledge } from
|
|
5
|
-
import { syncAttachmentChunk } from
|
|
1
|
+
import { hasLiveSource } from './routing.js';
|
|
2
|
+
import { removeChatAttachments, removeChatKnowledge } from '../../chat.js';
|
|
3
|
+
import { listChunks } from '../../chunk.js';
|
|
4
|
+
import { getKnowledge, listKnowledgeArtifactsByKnowledge } from './knowledge.js';
|
|
5
|
+
import { syncAttachmentChunk } from './chunks.js';
|
|
6
6
|
export async function syncKnowledgeTombstone(mesh, row) {
|
|
7
7
|
if (!row.deletedAt)
|
|
8
8
|
return;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { isSupportedRagDocument } from
|
|
2
|
-
import { groupAttachmentsByMessage } from
|
|
3
|
-
import { compareBlobReferences } from
|
|
4
|
-
import { getChat, setChatRagEnabled } from
|
|
5
|
-
import { listChunks } from
|
|
6
|
-
import { IMAGE_MIMES } from
|
|
7
|
-
import { addKnowledge, listKnowledge, removeKnowledge } from
|
|
1
|
+
import { isSupportedRagDocument } from '../rag/index.js';
|
|
2
|
+
import { groupAttachmentsByMessage } from './linkage.js';
|
|
3
|
+
import { compareBlobReferences } from '../store/index.js';
|
|
4
|
+
import { getChat, setChatRagEnabled } from '../../chat.js';
|
|
5
|
+
import { listChunks } from '../../chunk.js';
|
|
6
|
+
import { IMAGE_MIMES } from '../../image/normalize.js';
|
|
7
|
+
import { addKnowledge, listKnowledge, removeKnowledge } from './knowledge.js';
|
|
8
8
|
export function isKnowledgeRoutedAttachment(input) {
|
|
9
9
|
if (IMAGE_MIMES.has(input.mimeType.toLowerCase()))
|
|
10
10
|
return false;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
2
|
-
import { nextFrame } from
|
|
3
|
-
import { parseBlobReference } from
|
|
2
|
+
import { nextFrame } from '../../next-frame.js';
|
|
3
|
+
import { parseBlobReference } from './reference.js';
|
|
4
4
|
export async function* writeBlob(blobs, chunks, options = {}) {
|
|
5
5
|
const ws = blobs.createWriteStream();
|
|
6
6
|
const done = new Promise((resolve, reject) => {
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
2
|
import { Buffer } from 'buffer';
|
|
3
3
|
import EventEmitter from 'bare-events';
|
|
4
|
-
import { sweepUnreferenced } from
|
|
5
|
-
import { inlineReference } from
|
|
6
|
-
import { clearBlob, stringifyBlobId, writeBlob } from
|
|
7
|
-
import { parseBlobReference, readBlobReference, readBlobReferenceBytes, downloadBlobReference } from
|
|
8
|
-
export { CoreBlobServer } from
|
|
9
|
-
export { compareBlobReferences, isInvalidBlobReferenceError, parseBlobReference, parseBlobReferenceCoreKey } from
|
|
10
|
-
export { MAX_INLINE_BYTES, MAX_REFERENCE_CHARS } from
|
|
4
|
+
import { sweepUnreferenced } from './sweep.js';
|
|
5
|
+
import { inlineReference } from './inline.js';
|
|
6
|
+
import { clearBlob, stringifyBlobId, writeBlob } from './blob.js';
|
|
7
|
+
import { parseBlobReference, readBlobReference, readBlobReferenceBytes, downloadBlobReference } from './reference.js';
|
|
8
|
+
export { CoreBlobServer } from './server.js';
|
|
9
|
+
export { compareBlobReferences, isInvalidBlobReferenceError, parseBlobReference, parseBlobReferenceCoreKey } from './reference.js';
|
|
10
|
+
export { MAX_INLINE_BYTES, MAX_REFERENCE_CHARS } from './inline.js';
|
|
11
11
|
export class BlobStore extends EventEmitter {
|
|
12
12
|
blobs;
|
|
13
13
|
openBlobs;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { readBlob } from
|
|
1
|
+
import { readBlob } from './blob.js';
|
|
2
2
|
import { AbortedError, raceAbort } from '@qvac/harness/race-abort';
|
|
3
|
-
import { MAX_REFERENCE_CHARS, readInline } from
|
|
3
|
+
import { MAX_REFERENCE_CHARS, readInline } from './inline.js';
|
|
4
4
|
const CORE_KEY_PATTERN = /^[0-9a-f]{64}$/;
|
|
5
5
|
const DEFAULT_REMOTE_TIMEOUT_MS = 15_000;
|
|
6
6
|
const INVALID_BLOB_REFERENCE = 'invalid blob reference';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import BlobServer, {} from 'hypercore-blob-server';
|
|
2
2
|
import ReadyResource from 'ready-resource';
|
|
3
|
-
import { isInvalidBlobReferenceError, parseBlobReference } from
|
|
3
|
+
import { isInvalidBlobReferenceError, parseBlobReference } from './reference.js';
|
|
4
4
|
export class CoreBlobServer extends ReadyResource {
|
|
5
5
|
_store;
|
|
6
6
|
_mesh;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { parseBlobReference, parseBlobReferenceCoreKey } from
|
|
1
|
+
import { parseBlobReference, parseBlobReferenceCoreKey } from './reference.js';
|
|
2
2
|
export async function sweepUnreferenced(blobs, references, coreLength) {
|
|
3
3
|
const ranges = [];
|
|
4
4
|
const key = blobs.core.key.toString('hex');
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { AbortController } from 'bare-abort-controller';
|
|
2
|
-
import { toAsyncIterable } from
|
|
3
|
-
import { nextFrame } from
|
|
2
|
+
import { toAsyncIterable } from '../async-iterable.js';
|
|
3
|
+
import { nextFrame } from '../next-frame.js';
|
|
4
4
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
5
5
|
export class UploadInput {
|
|
6
6
|
iterator;
|
|
@@ -7,8 +7,8 @@ export declare function uploadFile(mesh: Mesh, frames: AsyncIterable<AssistantUp
|
|
|
7
7
|
deviceId?: Buffer | null;
|
|
8
8
|
inactivityMs?: number;
|
|
9
9
|
}): AsyncGenerator<{
|
|
10
|
-
bytesWritten: number;
|
|
11
10
|
attachment?: undefined;
|
|
11
|
+
bytesWritten: number;
|
|
12
12
|
knowledge?: undefined;
|
|
13
13
|
} | {
|
|
14
14
|
bytesWritten: number;
|