@dylanl1/dsh-plugin-pdf 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NATIVE_V1.md +3 -1
- package/README.md +54 -114
- package/README.zh.md +61 -121
- package/SPEC.md +46 -51
- package/dist/THIRD_PARTY_NOTICES.md +1 -56
- package/dist/client.js +5045 -2779
- package/dist/embedding.js +34 -9
- package/dist/engines.js +49 -4
- package/dist/image-host.js +837 -505
- package/dist/index.js +6790 -4126
- package/dist/ocr-host.js +1036 -175
- package/dist/ocr.js +4 -1
- package/dist/translation.js +43 -14
- package/dist/types/adapters/host/dsh/agent-tools.d.ts +1 -1
- package/dist/types/adapters/host/dsh/document-status.d.ts +0 -1
- package/dist/types/adapters/host/dsh/form-guidance.d.ts +2 -2
- package/dist/types/adapters/host/dsh/image-backends.d.ts +19 -3
- package/dist/types/adapters/host/dsh/image-tools.d.ts +15 -0
- package/dist/types/adapters/host/dsh/image-transport.d.ts +8 -7
- package/dist/types/adapters/host/dsh/index.d.ts +3 -2
- package/dist/types/adapters/host/dsh/model-cache-identity.d.ts +26 -0
- package/dist/types/adapters/host/dsh/ocr-session-llm.d.ts +2 -3
- package/dist/types/adapters/host/dsh/ocr-tool.d.ts +4 -2
- package/dist/types/adapters/host/dsh/page-content.d.ts +27 -0
- package/dist/types/adapters/host/dsh/pdf-evidence-meta.d.ts +13 -148
- package/dist/types/adapters/host/dsh/retrieval-prefetch.d.ts +49 -0
- package/dist/types/adapters/host/dsh/retrieval.d.ts +132 -25
- package/dist/types/adapters/host/dsh/translation-model.d.ts +3 -5
- package/dist/types/adapters/host/ocr/local-backend.d.ts +4 -0
- package/dist/types/contracts/indexing.d.ts +156 -0
- package/dist/types/embedding/registry.d.ts +2 -1
- package/dist/types/embedding/types.d.ts +10 -2
- package/dist/types/entrypoints/image-understanding-host.d.ts +2 -8
- package/dist/types/host/image-understanding-plugin.d.ts +1 -1
- package/dist/types/modules/document/index.d.ts +7 -0
- package/dist/types/modules/document/page-content-error.d.ts +3 -0
- package/dist/types/modules/document/page-content-native.d.ts +10 -0
- package/dist/types/modules/document/page-content-ocr.d.ts +5 -0
- package/dist/types/modules/document/page-content-text.d.ts +12 -0
- package/dist/types/modules/document/page-content.d.ts +158 -0
- package/dist/types/modules/engines/cache-identity.d.ts +16 -0
- package/dist/types/modules/engines/catalog.d.ts +20 -0
- package/dist/types/modules/engines/directory.d.ts +1 -0
- package/dist/types/modules/engines/index.d.ts +3 -0
- package/dist/types/modules/engines/serialization.d.ts +3 -0
- package/dist/types/modules/engines/types.d.ts +13 -0
- package/dist/types/modules/evidence/agent-page-image.d.ts +9 -112
- package/dist/types/modules/evidence/agent-page-read.d.ts +36 -2
- package/dist/types/modules/evidence/image-capture.d.ts +1 -6
- package/dist/types/modules/evidence/index.d.ts +9 -136
- package/dist/types/modules/evidence/provenance.d.ts +8 -0
- package/dist/types/modules/evidence/reference-store.d.ts +91 -1
- package/dist/types/modules/evidence/status.d.ts +17 -0
- package/dist/types/modules/evidence/text-range.d.ts +28 -0
- package/dist/types/modules/evidence/text-target.d.ts +7 -1
- package/dist/types/modules/forms/index.d.ts +4 -4
- package/dist/types/modules/forms/inspection-references.d.ts +1 -1
- package/dist/types/modules/forms/validation.d.ts +4 -8
- package/dist/types/modules/image-understanding/service.d.ts +24 -42
- package/dist/types/modules/model-write/index.d.ts +1 -2
- package/dist/types/modules/ocr/service.d.ts +51 -3
- package/dist/types/modules/retrieval/cache.d.ts +44 -2
- package/dist/types/modules/retrieval/document-policy.d.ts +83 -0
- package/dist/types/modules/retrieval/index.d.ts +4 -1
- package/dist/types/modules/retrieval/page-content.d.ts +4 -0
- package/dist/types/modules/retrieval/paragraphs.d.ts +1 -12
- package/dist/types/modules/retrieval/shared-source.d.ts +25 -0
- package/dist/types/modules/retrieval/text.d.ts +19 -1
- package/dist/types/modules/retrieval/types.d.ts +83 -9
- package/dist/types/modules/retrieval/vector.d.ts +3 -0
- package/dist/types/modules/working-copy/index.d.ts +1 -1
- package/dist/types/modules/working-copy/ports.d.ts +11 -0
- package/dist/types/modules/working-copy/service.d.ts +13 -1
- package/dist/types/ocr/types.d.ts +2 -1
- package/dist/types/shared/contracts.d.ts +4 -0
- package/dist/types/shared/geometry/native-viewport.d.ts +0 -18
- package/dist/types/shared/geometry/text-target.d.ts +2 -6
- package/dist/types/shared/native.d.ts +2 -1
- package/dist/types/shared/passive-poll.d.ts +4 -0
- package/dist/types/translation/locales.d.ts +4 -3
- package/dist/types/translation/schema.d.ts +14 -0
- package/dist/types/translation/types.d.ts +12 -2
- package/docs/AI_READER_TOOLS.md +87 -210
- package/docs/ARCHITECTURE_V2.md +64 -18
- package/docs/ENGINE_EXTENSIONS.md +295 -25
- package/docs/RETRIEVAL_EXPERIMENT.md +169 -47
- package/docs/UX_TYPOGRAPHY.md +3 -3
- package/package.json +13 -13
- package/dist/types/adapters/host/dsh/image-auxiliary.d.ts +0 -20
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import type { RetrievalCache } from './types.js';
|
|
3
|
-
export declare const RETRIEVAL_EXTRACTION_VERSION = "
|
|
3
|
+
export declare const RETRIEVAL_EXTRACTION_VERSION = "page-content-v3-representations";
|
|
4
4
|
export declare const digest: (value: string) => string;
|
|
5
5
|
export declare const retrievalCacheKey: (...values: unknown[]) => string;
|
|
6
6
|
declare const pageSchema: z.ZodObject<{
|
|
@@ -12,12 +12,54 @@ declare const pageSchema: z.ZodObject<{
|
|
|
12
12
|
}>;
|
|
13
13
|
warnings: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
14
14
|
paragraphBreaks: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
|
|
15
|
+
units: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
16
|
+
id: z.ZodString;
|
|
17
|
+
blockId: z.ZodString;
|
|
18
|
+
kind: z.ZodEnum<{
|
|
19
|
+
"ocr-text": "ocr-text";
|
|
20
|
+
"image-description": "image-description";
|
|
21
|
+
"native-text": "native-text";
|
|
22
|
+
}>;
|
|
23
|
+
text: z.ZodString;
|
|
24
|
+
digest: z.ZodString;
|
|
25
|
+
paragraphBreaks: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
|
|
26
|
+
nativeRange: z.ZodOptional<z.ZodObject<{
|
|
27
|
+
startUtf16: z.ZodNumber;
|
|
28
|
+
endUtf16: z.ZodNumber;
|
|
29
|
+
}, z.core.$strict>>;
|
|
30
|
+
rectPdf: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber, z.ZodNumber, z.ZodNumber], null>>;
|
|
31
|
+
spans: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
32
|
+
startUtf16: z.ZodNumber;
|
|
33
|
+
endUtf16: z.ZodNumber;
|
|
34
|
+
rectPdf: z.ZodTuple<[z.ZodNumber, z.ZodNumber, z.ZodNumber, z.ZodNumber], null>;
|
|
35
|
+
}, z.core.$strict>>>;
|
|
36
|
+
producer: z.ZodOptional<z.ZodObject<{
|
|
37
|
+
id: z.ZodString;
|
|
38
|
+
version: z.ZodOptional<z.ZodString>;
|
|
39
|
+
model: z.ZodOptional<z.ZodString>;
|
|
40
|
+
configurationIdentity: z.ZodOptional<z.ZodString>;
|
|
41
|
+
}, z.core.$strict>>;
|
|
42
|
+
inputDigest: z.ZodOptional<z.ZodString>;
|
|
43
|
+
}, z.core.$strict>>>;
|
|
44
|
+
contentIdentity: z.ZodOptional<z.ZodString>;
|
|
15
45
|
}, z.core.$strict>;
|
|
16
46
|
export type CachedRetrievalPage = z.infer<typeof pageSchema>;
|
|
17
|
-
|
|
47
|
+
/** The empty legacy identity intentionally preserves the unextended key shape. */
|
|
48
|
+
export declare function pageCacheKey(path: string, sourceDigest: string, page: number, extractionIdentity?: string): string;
|
|
49
|
+
export declare function validateRetrievalPage(value: unknown): CachedRetrievalPage | undefined;
|
|
18
50
|
export declare function readCachedPage(cache: RetrievalCache | undefined, key: string): Promise<CachedRetrievalPage | undefined>;
|
|
19
51
|
/** Derived caches are disposable; an unavailable cache must not break reading. */
|
|
20
52
|
export declare function cacheDerived(cache: RetrievalCache | undefined, key: string, value: unknown): Promise<void>;
|
|
21
53
|
export declare function validVector(value: unknown, dimensions: number): value is number[];
|
|
54
|
+
interface CachedRetrievalVector {
|
|
55
|
+
version: 1;
|
|
56
|
+
encoding: 'float32-le-base64';
|
|
57
|
+
dimensions: number;
|
|
58
|
+
data: string;
|
|
59
|
+
sha256: string;
|
|
60
|
+
}
|
|
61
|
+
/** Persist the same normalized Float32 precision used by cosine search. Explicit
|
|
62
|
+
* byte order also makes caches reusable between CPU architectures. */
|
|
63
|
+
export declare function encodeCachedVector(input: ArrayLike<number>): CachedRetrievalVector;
|
|
22
64
|
export declare function readCachedVector(cache: RetrievalCache | undefined, key: string, dimensions: number): Promise<number[] | undefined>;
|
|
23
65
|
export {};
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { type PdfDocumentIndexPolicy, type PdfDocumentIndexSettings } from '../../contracts/indexing.js';
|
|
3
|
+
import { type EngineJson } from '../engines/index.js';
|
|
4
|
+
import { type PdfSettings } from '../../shared/contracts.js';
|
|
5
|
+
export declare const resolvedIndexPlanSchema: z.ZodObject<{
|
|
6
|
+
policy: z.ZodObject<{
|
|
7
|
+
indexScanPages: z.ZodBoolean;
|
|
8
|
+
indexImageDescriptions: z.ZodBoolean;
|
|
9
|
+
embeddingEngine: z.ZodString;
|
|
10
|
+
ocrEngine: z.ZodString;
|
|
11
|
+
imageEngine: z.ZodString;
|
|
12
|
+
}, z.core.$strict>;
|
|
13
|
+
configurations: z.ZodObject<{
|
|
14
|
+
ocr: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<EngineJson, unknown, z.core.$ZodTypeInternals<EngineJson, unknown>>>>>;
|
|
15
|
+
image: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<EngineJson, unknown, z.core.$ZodTypeInternals<EngineJson, unknown>>>>>;
|
|
16
|
+
translation: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<EngineJson, unknown, z.core.$ZodTypeInternals<EngineJson, unknown>>>>>;
|
|
17
|
+
embedding: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<EngineJson, unknown, z.core.$ZodTypeInternals<EngineJson, unknown>>>>>;
|
|
18
|
+
}, z.core.$strict>;
|
|
19
|
+
}, z.core.$strict>;
|
|
20
|
+
export type ResolvedIndexPlan = z.infer<typeof resolvedIndexPlanSchema>;
|
|
21
|
+
export declare const documentIndexRecordSchema: z.ZodObject<{
|
|
22
|
+
version: z.ZodOptional<z.ZodLiteral<2>>;
|
|
23
|
+
revision: z.ZodNumber;
|
|
24
|
+
desired: z.ZodUnion<readonly [z.ZodObject<{
|
|
25
|
+
indexScanPages: z.ZodBoolean;
|
|
26
|
+
indexImageDescriptions: z.ZodBoolean;
|
|
27
|
+
}, z.core.$strict>, z.ZodPipe<z.ZodObject<{
|
|
28
|
+
indexScanPages: z.ZodBoolean;
|
|
29
|
+
indexImageDescriptions: z.ZodBoolean;
|
|
30
|
+
embeddingEngine: z.ZodString;
|
|
31
|
+
ocrEngine: z.ZodString;
|
|
32
|
+
imageEngine: z.ZodString;
|
|
33
|
+
}, z.core.$strict>, z.ZodTransform<{
|
|
34
|
+
indexScanPages: boolean;
|
|
35
|
+
indexImageDescriptions: boolean;
|
|
36
|
+
}, {
|
|
37
|
+
indexScanPages: boolean;
|
|
38
|
+
indexImageDescriptions: boolean;
|
|
39
|
+
embeddingEngine: string;
|
|
40
|
+
ocrEngine: string;
|
|
41
|
+
imageEngine: string;
|
|
42
|
+
}>>]>;
|
|
43
|
+
active: z.ZodObject<{
|
|
44
|
+
policy: z.ZodObject<{
|
|
45
|
+
indexScanPages: z.ZodBoolean;
|
|
46
|
+
indexImageDescriptions: z.ZodBoolean;
|
|
47
|
+
embeddingEngine: z.ZodString;
|
|
48
|
+
ocrEngine: z.ZodString;
|
|
49
|
+
imageEngine: z.ZodString;
|
|
50
|
+
}, z.core.$strict>;
|
|
51
|
+
configurations: z.ZodObject<{
|
|
52
|
+
ocr: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<EngineJson, unknown, z.core.$ZodTypeInternals<EngineJson, unknown>>>>>;
|
|
53
|
+
image: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<EngineJson, unknown, z.core.$ZodTypeInternals<EngineJson, unknown>>>>>;
|
|
54
|
+
translation: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<EngineJson, unknown, z.core.$ZodTypeInternals<EngineJson, unknown>>>>>;
|
|
55
|
+
embedding: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<EngineJson, unknown, z.core.$ZodTypeInternals<EngineJson, unknown>>>>>;
|
|
56
|
+
}, z.core.$strict>;
|
|
57
|
+
}, z.core.$strict>;
|
|
58
|
+
}, z.core.$strict>;
|
|
59
|
+
type Record = z.infer<typeof documentIndexRecordSchema>;
|
|
60
|
+
interface Store {
|
|
61
|
+
key(path: string): string;
|
|
62
|
+
get(key: string): unknown;
|
|
63
|
+
put(key: string, value: Record): Promise<unknown>;
|
|
64
|
+
}
|
|
65
|
+
export declare function indexPolicy(settings: Pick<PdfSettings, 'indexScanPages' | 'indexImageDescriptions'>): PdfDocumentIndexPolicy;
|
|
66
|
+
/** Freeze only the selected bindings; edits to unrelated engines do not invalidate this document. */
|
|
67
|
+
export declare function resolveIndexPlan(policy: PdfDocumentIndexPolicy, settings: PdfSettings): ResolvedIndexPlan;
|
|
68
|
+
export declare function settingsForIndexPlan(plan: ResolvedIndexPlan, defaults?: PdfSettings): PdfSettings;
|
|
69
|
+
/** Host/profile-owned preferences. The path key survives saves; source digests belong to derived indexes. */
|
|
70
|
+
export declare function createDocumentIndexPolicies(store: Store, defaults: () => PdfSettings): {
|
|
71
|
+
enabled: () => boolean;
|
|
72
|
+
active(path: string): ResolvedIndexPlan;
|
|
73
|
+
desired: (path: string) => ResolvedIndexPlan;
|
|
74
|
+
settled(path: string): Promise<void>;
|
|
75
|
+
snapshot(path: string, documentId: string): PdfDocumentIndexSettings;
|
|
76
|
+
configure(path: string, revision: number, policy: PdfDocumentIndexPolicy): Promise<void>;
|
|
77
|
+
defaults(path: string, revision: number): Promise<void>;
|
|
78
|
+
activate(path: string, plan: ResolvedIndexPlan): Promise<void>;
|
|
79
|
+
subscribe(listener: () => void): () => void;
|
|
80
|
+
releaseUnused(paths: ReadonlySet<string>): void;
|
|
81
|
+
dispose(): Promise<void>;
|
|
82
|
+
};
|
|
83
|
+
export {};
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
export * from './types.js';
|
|
2
2
|
export * from './service.js';
|
|
3
|
-
export {
|
|
3
|
+
export { createDocumentIndexPolicies, documentIndexRecordSchema, indexPolicy, resolveIndexPlan, settingsForIndexPlan } from './document-policy.js';
|
|
4
|
+
export type { ResolvedIndexPlan } from './document-policy.js';
|
|
5
|
+
export { chunkPage, chunkRetrievalPage, tokenizeRetrievalText, RETRIEVAL_TEXT_VERSION } from './text.js';
|
|
4
6
|
export { retrievalPageText } from './paragraphs.js';
|
|
5
7
|
export type { RetrievalChunk } from './text.js';
|
|
6
8
|
export { createBm25Index, searchBm25, rankVectors, reciprocalRankFusion, hybridCandidates } from './ranking.js';
|
|
9
|
+
export { projectPageContent } from './page-content.js';
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { type PageContent } from '../document/index.js';
|
|
2
|
+
import type { RetrievalPage } from './types.js';
|
|
3
|
+
/** Indexing is a projection of representations, not a second editable document. */
|
|
4
|
+
export declare function projectPageContent(content: PageContent): RetrievalPage;
|
|
@@ -1,12 +1 @@
|
|
|
1
|
-
|
|
2
|
-
type TextItem = NativeTextContent['items'][number];
|
|
3
|
-
/**
|
|
4
|
-
* Preserve native extraction order and exact joined-text offsets. Only obvious
|
|
5
|
-
* layout gaps/font changes add boundaries; ordinary wrapped lines remain one
|
|
6
|
-
* paragraph. These are retrieval units, never certified reading-order geometry.
|
|
7
|
-
*/
|
|
8
|
-
export declare function retrievalPageText(items: readonly TextItem[]): {
|
|
9
|
-
text: string;
|
|
10
|
-
paragraphBreaks: number[];
|
|
11
|
-
};
|
|
12
|
-
export {};
|
|
1
|
+
export { nativePageContentText as retrievalPageText } from '../document/index.js';
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { RetrievalDocument, RetrievalEmbedding, RetrievalPorts } from './types.js';
|
|
2
|
+
/** Authority belongs to a caller, never to the content-addressed index. */
|
|
3
|
+
export interface RetrievalBinding {
|
|
4
|
+
sessionId: string;
|
|
5
|
+
document: RetrievalDocument;
|
|
6
|
+
provider?: RetrievalEmbedding;
|
|
7
|
+
selectionIdentity: string;
|
|
8
|
+
extractionIdentity: string;
|
|
9
|
+
lifetime: {
|
|
10
|
+
valid: boolean;
|
|
11
|
+
id: string;
|
|
12
|
+
};
|
|
13
|
+
signal: AbortSignal;
|
|
14
|
+
}
|
|
15
|
+
/** A shared step owns its lease. A departing caller does not cancel another
|
|
16
|
+
* caller's work; a changed/revoked source is replaced before derived data commits. */
|
|
17
|
+
export declare function createSharedRetrievalSource(ports: RetrievalPorts, bindings: ReadonlySet<RetrievalBinding>, signal: AbortSignal, checkIndex: () => void, checkBinding: (binding: RetrievalBinding) => void, checkOrigin: (binding: RetrievalBinding) => void): {
|
|
18
|
+
current: (excluded?: ReadonlySet<RetrievalBinding>) => RetrievalBinding;
|
|
19
|
+
check: () => void;
|
|
20
|
+
read: (page: number) => Promise<import("./types.js").RetrievalPage>;
|
|
21
|
+
embed: (texts: string[]) => Promise<number[][]>;
|
|
22
|
+
signal: AbortSignal;
|
|
23
|
+
dispose: () => void;
|
|
24
|
+
};
|
|
25
|
+
export type SharedRetrievalSource = ReturnType<typeof createSharedRetrievalSource>;
|
|
@@ -1,14 +1,29 @@
|
|
|
1
|
+
import type { RetrievalPage, RetrievalPageUnitKind, RetrievalPageUnitProducer, RetrievalPageUnitSpan } from './types.js';
|
|
1
2
|
/** Changing boundaries or tokenization invalidates the derived lexical index. */
|
|
2
3
|
export declare const RETRIEVAL_TEXT_VERSION = "page-text-v2-paragraphs";
|
|
3
4
|
export declare const MAX_RETRIEVAL_PARAGRAPH_BREAKS = 10000;
|
|
4
5
|
export interface RetrievalChunk {
|
|
5
6
|
id: string;
|
|
6
7
|
page: number;
|
|
8
|
+
/** Joined native-page offsets for pdf-text; representation-local offsets for generated content. */
|
|
7
9
|
startUtf16: number;
|
|
8
10
|
endUtf16: number;
|
|
9
11
|
text: string;
|
|
12
|
+
/** Native joined-page digest, or exact OCR/description unit text digest (unprefixed SHA-256). */
|
|
10
13
|
pageTextDigest: string;
|
|
11
|
-
source: 'pdf-text' | 'ocr';
|
|
14
|
+
source: 'pdf-text' | 'ocr' | 'image-description';
|
|
15
|
+
representation?: {
|
|
16
|
+
id: string;
|
|
17
|
+
blockId: string;
|
|
18
|
+
kind: RetrievalPageUnitKind;
|
|
19
|
+
unitDigest: string;
|
|
20
|
+
unitStartUtf16: number;
|
|
21
|
+
unitEndUtf16: number;
|
|
22
|
+
rectPdf?: [number, number, number, number];
|
|
23
|
+
spans?: RetrievalPageUnitSpan[];
|
|
24
|
+
producer?: RetrievalPageUnitProducer;
|
|
25
|
+
inputDigest?: string;
|
|
26
|
+
};
|
|
12
27
|
}
|
|
13
28
|
export interface ChunkPageOptions {
|
|
14
29
|
maxCharacters?: number;
|
|
@@ -21,3 +36,6 @@ export declare function tokenizeRetrievalText(text: string): string[];
|
|
|
21
36
|
export declare function validParagraphBreaks(text: string, breaks: unknown): breaks is number[] | undefined;
|
|
22
37
|
/** Page-local exact substrings. UTF-16 offsets refer to the unmodified joined page text. */
|
|
23
38
|
export declare function chunkPage(page: number, text: string, source?: RetrievalChunk['source'], options?: ChunkPageOptions): RetrievalChunk[];
|
|
39
|
+
/** Chunk the projected representations without losing their evidence coordinate
|
|
40
|
+
* system. Native ranges refer to joined page text; generated ranges are local. */
|
|
41
|
+
export declare function chunkRetrievalPage(page: number, content: RetrievalPage, options?: Pick<ChunkPageOptions, 'maxCharacters' | 'overlapCharacters'>): RetrievalChunk[];
|
|
@@ -10,8 +10,13 @@ export interface RetrievalDocument {
|
|
|
10
10
|
export interface RetrievalEmbedding {
|
|
11
11
|
/** Includes provider, model revision, dimensions and query/document encoding configuration. */
|
|
12
12
|
fingerprint: string;
|
|
13
|
+
/** Provider-attested document encoding identity. When absent the complete
|
|
14
|
+
* fingerprint/binding pair remains the conservative persistent vector key. */
|
|
15
|
+
documentFingerprint?: string;
|
|
13
16
|
/** Registration lifetime, separate from the reusable model/config fingerprint. */
|
|
14
17
|
generation?: string;
|
|
18
|
+
/** Exact engine binding identity; third-party fingerprints may omit configuration. */
|
|
19
|
+
configurationIdentity?: string;
|
|
15
20
|
dimensions: number;
|
|
16
21
|
maxBatchSize: number;
|
|
17
22
|
maxInputCharacters?: number;
|
|
@@ -21,6 +26,55 @@ export interface RetrievalEmbedding {
|
|
|
21
26
|
export interface RetrievalCache {
|
|
22
27
|
get(key: string): Promise<unknown>;
|
|
23
28
|
set(key: string, value: unknown): Promise<void>;
|
|
29
|
+
/** Remove all derived records. Required when this cache supports explicit rebuilds. */
|
|
30
|
+
clear?(): Promise<void>;
|
|
31
|
+
}
|
|
32
|
+
export type RetrievalPageUnitKind = 'native-text' | 'ocr-text' | 'image-description';
|
|
33
|
+
export interface RetrievalPageUnitProducer {
|
|
34
|
+
id: string;
|
|
35
|
+
version?: string;
|
|
36
|
+
model?: string;
|
|
37
|
+
configurationIdentity?: string;
|
|
38
|
+
}
|
|
39
|
+
export interface RetrievalPageUnitSpan {
|
|
40
|
+
startUtf16: number;
|
|
41
|
+
endUtf16: number;
|
|
42
|
+
rectPdf: [number, number, number, number];
|
|
43
|
+
}
|
|
44
|
+
/** One exact representation of a logical content block. Generated offsets stay
|
|
45
|
+
* local to this unit and must never be interpreted as native PDF text offsets. */
|
|
46
|
+
export interface RetrievalPageUnit {
|
|
47
|
+
id: string;
|
|
48
|
+
blockId: string;
|
|
49
|
+
kind: RetrievalPageUnitKind;
|
|
50
|
+
text: string;
|
|
51
|
+
/** SHA-256 of the exact representation text, prefixed with sha256:. */
|
|
52
|
+
digest: string;
|
|
53
|
+
paragraphBreaks?: number[];
|
|
54
|
+
nativeRange?: {
|
|
55
|
+
startUtf16: number;
|
|
56
|
+
endUtf16: number;
|
|
57
|
+
};
|
|
58
|
+
rectPdf?: [number, number, number, number];
|
|
59
|
+
spans?: RetrievalPageUnitSpan[];
|
|
60
|
+
producer?: RetrievalPageUnitProducer;
|
|
61
|
+
inputDigest?: string;
|
|
62
|
+
}
|
|
63
|
+
export interface RetrievalPageFailure {
|
|
64
|
+
code: string;
|
|
65
|
+
}
|
|
66
|
+
export interface RetrievalPage {
|
|
67
|
+
/** Joined native text; legacy OCR-only ports may keep their existing format. */
|
|
68
|
+
text: string;
|
|
69
|
+
source: 'pdf-text' | 'ocr';
|
|
70
|
+
warnings?: string[];
|
|
71
|
+
paragraphBreaks?: number[];
|
|
72
|
+
/** When supplied, these are the complete searchable units for this page. */
|
|
73
|
+
units?: RetrievalPageUnit[];
|
|
74
|
+
contentIdentity?: string;
|
|
75
|
+
/** Available representations are usable, but optional extraction was incomplete.
|
|
76
|
+
* Such pages are never persisted and retry only through an explicit rebuild. */
|
|
77
|
+
failure?: RetrievalPageFailure;
|
|
24
78
|
}
|
|
25
79
|
export interface RetrievalPorts {
|
|
26
80
|
documents: {
|
|
@@ -29,13 +83,9 @@ export interface RetrievalPorts {
|
|
|
29
83
|
retain?(sessionId: string, documentId: string, signal: AbortSignal): Promise<() => void>;
|
|
30
84
|
};
|
|
31
85
|
pages: {
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
warnings?: string[];
|
|
36
|
-
/** Source UTF-16 offsets of conservatively identified paragraph starts. */
|
|
37
|
-
paragraphBreaks?: number[];
|
|
38
|
-
}>;
|
|
86
|
+
/** Synchronous extraction/pipeline identity, independent of embedding selection. */
|
|
87
|
+
identity?(sessionId: string): string;
|
|
88
|
+
read(sessionId: string, documentId: string, page: number, signal: AbortSignal): Promise<RetrievalPage>;
|
|
39
89
|
};
|
|
40
90
|
cache?: RetrievalCache;
|
|
41
91
|
embedding?: {
|
|
@@ -62,6 +112,22 @@ export interface RetrievalInput {
|
|
|
62
112
|
maxPages?: number;
|
|
63
113
|
cursor?: string;
|
|
64
114
|
}
|
|
115
|
+
/** Query-free preparation shares the same scoped index as later searches. */
|
|
116
|
+
export type RetrievalPrepareInput = Pick<RetrievalInput, 'documentId' | 'fromPage' | 'maxPages'> & {
|
|
117
|
+
/** Re-extract and re-embed this source with the selected engine, preserving unrelated indexes. */
|
|
118
|
+
rebuild?: boolean;
|
|
119
|
+
};
|
|
120
|
+
/** Observable work counts, separate from task completion and document authority. */
|
|
121
|
+
export interface RetrievalProgress {
|
|
122
|
+
indexedPages: number;
|
|
123
|
+
totalPages: number;
|
|
124
|
+
embeddedChunks: number;
|
|
125
|
+
totalChunks: number;
|
|
126
|
+
/** Current work, distinct from counters that also advance on cache hits. */
|
|
127
|
+
phase?: 'restoring' | 'extracting' | 'embedding' | 'ready';
|
|
128
|
+
reusedChunks?: number;
|
|
129
|
+
computedChunks?: number;
|
|
130
|
+
}
|
|
65
131
|
export interface RetrievalResult {
|
|
66
132
|
status: 'building' | 'complete' | 'incomplete';
|
|
67
133
|
requestedMode: RetrievalMode;
|
|
@@ -76,8 +142,10 @@ export interface RetrievalResult {
|
|
|
76
142
|
coverage: {
|
|
77
143
|
indexedPages: number;
|
|
78
144
|
totalPages: number;
|
|
145
|
+
/** No native or OCR text; a generated image description does not imply page transcription. */
|
|
79
146
|
noTextPages: number[];
|
|
80
147
|
ocrPages: number[];
|
|
148
|
+
imageDescriptionPages: number[];
|
|
81
149
|
failedPages: {
|
|
82
150
|
page: number;
|
|
83
151
|
attempts: number;
|
|
@@ -89,18 +157,24 @@ export interface RetrievalResult {
|
|
|
89
157
|
}[];
|
|
90
158
|
embeddedChunks: number;
|
|
91
159
|
totalChunks: number;
|
|
92
|
-
visual: 'not-indexed';
|
|
160
|
+
visual: 'not-indexed' | 'image-descriptions';
|
|
93
161
|
};
|
|
94
|
-
/**
|
|
162
|
+
/** Native offsets support exact evidence; generated units require their own
|
|
163
|
+
* representation provenance and approximate PDF region evidence. */
|
|
95
164
|
chunks: (RetrievalChunk & {
|
|
96
165
|
score: number;
|
|
97
166
|
})[];
|
|
98
167
|
nextCursor?: string;
|
|
99
168
|
}
|
|
100
169
|
export interface RetrievalService {
|
|
170
|
+
/** Operational state only; never a document access or engine capability grant. */
|
|
171
|
+
state(): 'ready' | 'rebuilding' | 'failed' | 'disposed';
|
|
172
|
+
prepare(sessionId: string, input: RetrievalPrepareInput, signal?: AbortSignal, onProgress?: (progress: RetrievalProgress) => void): Promise<void>;
|
|
101
173
|
search(sessionId: string, input: RetrievalInput, signal?: AbortSignal): Promise<RetrievalResult>;
|
|
102
174
|
/** Revalidate after adapter awaits, including reference issuance; never an authorization grant. */
|
|
103
175
|
validate(sessionId: string, result: RetrievalResult): void;
|
|
104
176
|
cancelSession(sessionId: string): void;
|
|
177
|
+
/** Invalidate active searches and derived caches; later searches rebuild on demand. */
|
|
178
|
+
rebuild(): Promise<void>;
|
|
105
179
|
dispose(): Promise<void>;
|
|
106
180
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type { HostWorkspaceSnapshot, WorkingCopyReadPort, DocumentResidency, DocumentReleaseStatus } from './ports.js';
|
|
1
|
+
export type { HostWorkspaceSnapshot, WorkingCopyReadPort, DocumentResidency, DocumentReleaseStatus, OpenWorkingCopyDocument } from './ports.js';
|
|
2
2
|
export { createWorkingCopyService } from './service.js';
|
|
3
3
|
export type { PdfDocumentPort, V1DraftPort, WorkingCopyDraftPort, WorkingCopyAgent, WorkingCopyFilePort, WorkingCopyFileRead, } from './ports.js';
|
|
4
4
|
export type { DraftRecord, WorkspaceRecord } from './schema.js';
|
|
@@ -18,6 +18,17 @@ export interface DocumentResidency {
|
|
|
18
18
|
modelReferenceId?: string;
|
|
19
19
|
pins: number;
|
|
20
20
|
}
|
|
21
|
+
/** Open ownership and source identity only; operation pins never constitute an open document. */
|
|
22
|
+
export interface OpenWorkingCopyDocument {
|
|
23
|
+
sessionId: string;
|
|
24
|
+
id: string;
|
|
25
|
+
/** Canonical resource identity remains available after idle working-set eviction. */
|
|
26
|
+
path?: string;
|
|
27
|
+
contentVersion: string;
|
|
28
|
+
sourceDigest: string;
|
|
29
|
+
conflict: boolean;
|
|
30
|
+
title?: string;
|
|
31
|
+
}
|
|
21
32
|
/** Minimum authority needed by read-only features after session authorization. */
|
|
22
33
|
export interface WorkingCopyReadPort {
|
|
23
34
|
readSnapshot(sessionId: string, documentId: string): HostWorkspaceSnapshot;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { PdfDocumentInfo } from '../../core/pdf-types.js';
|
|
2
|
+
import type { HostWorkspaceSnapshot, PdfDocumentPort, WorkingCopyDraftPort, WorkingCopyAgent, WorkingCopyFilePort, DocumentReleaseStatus, DocumentResidency, OpenWorkingCopyDocument } from './ports.js';
|
|
2
3
|
export type { HostWorkspaceSnapshot } from './ports.js';
|
|
3
4
|
/** Authoritative working copies, serialized per session. A draft is durable before the UI accepts it. */
|
|
4
5
|
export declare function createWorkingCopyService(files: WorkingCopyFilePort, drafts: WorkingCopyDraftPort, pdf: PdfDocumentPort): {
|
|
@@ -30,10 +31,21 @@ export declare function createWorkingCopyService(files: WorkingCopyFilePort, dra
|
|
|
30
31
|
id?: undefined;
|
|
31
32
|
status?: undefined;
|
|
32
33
|
}>;
|
|
34
|
+
/** Inspect source bytes without acquiring an owner or creating a durable draft. */
|
|
35
|
+
inspectFile(sessionId: string, path: string, agent: WorkingCopyAgent, signal: AbortSignal): Promise<{
|
|
36
|
+
path: string;
|
|
37
|
+
size: number;
|
|
38
|
+
sourceVersion: string;
|
|
39
|
+
sourceDigest: string;
|
|
40
|
+
document: PdfDocumentInfo;
|
|
41
|
+
}>;
|
|
33
42
|
/** Pin the whole operation, including image decoding or save approval waits. */
|
|
34
43
|
acquire(sessionId: string, id: string, agent: WorkingCopyAgent, signal: AbortSignal): Promise<() => void>;
|
|
35
44
|
touch(sessionId: string, id: string): void;
|
|
36
45
|
onIdle(listener: (sessionId: string, id: string) => void): () => void;
|
|
46
|
+
listOpenDocuments: () => OpenWorkingCopyDocument[];
|
|
47
|
+
/** Passive ownership/source notifications; subscribers schedule their own asynchronous work. */
|
|
48
|
+
onDocumentsChanged(listener: () => void): () => void;
|
|
37
49
|
listDocuments(): DocumentResidency[];
|
|
38
50
|
waitForRevision(sessionId: string, id: string, knownRevision: number, signal: AbortSignal): Promise<number>;
|
|
39
51
|
isUiOwned(sessionId: string, id: string): boolean;
|
|
@@ -59,7 +59,8 @@ export interface OcrEngineDescriptor {
|
|
|
59
59
|
id: string;
|
|
60
60
|
name: string;
|
|
61
61
|
version: string;
|
|
62
|
-
execution: '
|
|
62
|
+
execution: import('../modules/engines/types.js').EngineDescriptor['execution'];
|
|
63
|
+
capabilities?: import('../modules/engines/types.js').EngineCapability[];
|
|
63
64
|
languages?: readonly string[];
|
|
64
65
|
geometry: readonly ('block' | 'line' | 'word')[];
|
|
65
66
|
confidence: boolean;
|
|
@@ -21,6 +21,10 @@ export interface PdfSettings {
|
|
|
21
21
|
ocrEngine?: string;
|
|
22
22
|
imageEngine?: string;
|
|
23
23
|
embeddingEngine?: string;
|
|
24
|
+
/** Recognize scanned or damaged pages with the selected OCR engine while indexing. */
|
|
25
|
+
indexScanPages?: boolean;
|
|
26
|
+
/** Optional generated whole-page visual descriptions, independently of OCR. */
|
|
27
|
+
indexImageDescriptions?: boolean;
|
|
24
28
|
engineConfigurations?: import('../modules/engines/types.js').EngineConfigurations;
|
|
25
29
|
ocrTimeoutMs: number;
|
|
26
30
|
translationEngine: string;
|
|
@@ -1,31 +1,13 @@
|
|
|
1
1
|
import type { PdfPageInfo } from '../../core/pdf-types.js';
|
|
2
2
|
/** PDF user space to CSS pixels; crop, UserUnit, rotation and Y inversion occur once. */
|
|
3
3
|
export declare class NativeViewport {
|
|
4
|
-
readonly geometry: PdfPageInfo;
|
|
5
4
|
readonly scale: number;
|
|
6
5
|
readonly rotation: number;
|
|
7
|
-
readonly offsetX: number;
|
|
8
|
-
readonly offsetY: number;
|
|
9
|
-
readonly dontFlip: boolean;
|
|
10
|
-
readonly viewBox: number[];
|
|
11
6
|
readonly userUnit: number;
|
|
12
7
|
readonly width: number;
|
|
13
8
|
readonly height: number;
|
|
14
9
|
readonly transform: number[];
|
|
15
|
-
readonly rawDims: {
|
|
16
|
-
pageWidth: number;
|
|
17
|
-
pageHeight: number;
|
|
18
|
-
pageX: number;
|
|
19
|
-
pageY: number;
|
|
20
|
-
};
|
|
21
10
|
constructor(geometry: PdfPageInfo, scale: number, rotation?: number, offsetX?: number, offsetY?: number, dontFlip?: boolean);
|
|
22
|
-
clone(options?: {
|
|
23
|
-
scale?: number;
|
|
24
|
-
rotation?: number;
|
|
25
|
-
offsetX?: number;
|
|
26
|
-
offsetY?: number;
|
|
27
|
-
dontFlip?: boolean;
|
|
28
|
-
}): NativeViewport;
|
|
29
11
|
convertToViewportPoint(x: number, y: number): [number, number];
|
|
30
12
|
convertToPdfPoint(x: number, y: number): [number, number];
|
|
31
13
|
convertToViewportRectangle(rect: number[]): number[];
|
|
@@ -1,17 +1,13 @@
|
|
|
1
1
|
import type { PdfRect } from '../../core/pdf-types.js';
|
|
2
2
|
import type { NativeTextContent } from '../native.js';
|
|
3
|
-
export interface
|
|
3
|
+
export interface TextTargetRange {
|
|
4
4
|
rect: PdfRect;
|
|
5
5
|
quadPoints: number[];
|
|
6
|
-
}
|
|
7
|
-
export interface TextTargetRange extends TextTargetGeometry {
|
|
8
6
|
startUtf16: number;
|
|
9
7
|
endUtf16: number;
|
|
10
8
|
text: string;
|
|
11
9
|
}
|
|
12
|
-
/** Resolve bounded start/end anchors against the same joined text exposed by
|
|
10
|
+
/** Resolve bounded start/end anchors against the same joined text exposed by pdf_page_read. */
|
|
13
11
|
export declare function locateTextRange(content: NativeTextContent, startQuote?: string, endQuote?: string, requireReadingOrder?: boolean): TextTargetRange;
|
|
14
12
|
/** Resolve an attested UTF-16 range, including repeated quotes, against native glyphs only. */
|
|
15
13
|
export declare function locateTextOffsetRange(content: NativeTextContent, start: number, end: number, requireReadingOrder?: boolean): TextTargetRange;
|
|
16
|
-
/** Resolve an exact, unique quote to PDF-space glyph geometry, never to a guessed page rectangle. */
|
|
17
|
-
export declare function locateTextTarget(content: NativeTextContent, quote: string): TextTargetGeometry;
|
|
@@ -29,7 +29,8 @@ export interface NativeTextContent {
|
|
|
29
29
|
dir: string;
|
|
30
30
|
fontName: string;
|
|
31
31
|
hasEOL: boolean;
|
|
32
|
-
/** PDF-space origin
|
|
32
|
+
/** One PDF-space origin/advance per Unicode scalar, including single-scalar runs.
|
|
33
|
+
* Omitted only when native positioning is unavailable or the page exceeds the frame budget. */
|
|
33
34
|
glyphs?: [x: number, y: number, advance: number][];
|
|
34
35
|
}[];
|
|
35
36
|
styles: Record<string, {
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** Compatibility with official Desktop builds whose update admission check expires in 10 seconds.
|
|
2
|
+
* Read-only subscriptions yield with 5 seconds of headroom; events still wake them immediately.
|
|
3
|
+
* This budget never bounds writes or active tool work. */
|
|
4
|
+
export declare const PASSIVE_POLL_TIMEOUT_MS = 5000;
|
|
@@ -11,7 +11,10 @@ export declare const translationEn: {
|
|
|
11
11
|
'translation.retry': string;
|
|
12
12
|
'translation.cancel': string;
|
|
13
13
|
'translation.original': string;
|
|
14
|
-
'translation.
|
|
14
|
+
'translation.interpretationCorrection': string;
|
|
15
|
+
'translation.interpretationCompletion': string;
|
|
16
|
+
'translation.interpretationPossibleCorrection': string;
|
|
17
|
+
'translation.interpretationPossibleCompletion': string;
|
|
15
18
|
'translation.loading': string;
|
|
16
19
|
'translation.cancelled': string;
|
|
17
20
|
'translation.result': string;
|
|
@@ -48,7 +51,5 @@ export declare const translationEn: {
|
|
|
48
51
|
'translation.errorInvalidModelOutput': string;
|
|
49
52
|
'translation.errorIncompleteModelOutput': string;
|
|
50
53
|
'translation.errorEmptyModelOutput': string;
|
|
51
|
-
'translation.errorIncompleteSelection': string;
|
|
52
|
-
'translation.errorUnrecognizedSelection': string;
|
|
53
54
|
};
|
|
54
55
|
export declare const translationZh: Record<keyof typeof translationEn, string>;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/** Shared by model parsing, engine validation and the authenticated Client wire. */
|
|
3
|
+
export declare const interpretationSchema: z.ZodObject<{
|
|
4
|
+
text: z.ZodString;
|
|
5
|
+
kind: z.ZodEnum<{
|
|
6
|
+
correction: "correction";
|
|
7
|
+
completion: "completion";
|
|
8
|
+
}>;
|
|
9
|
+
certainty: z.ZodEnum<{
|
|
10
|
+
confident: "confident";
|
|
11
|
+
tentative: "tentative";
|
|
12
|
+
}>;
|
|
13
|
+
explanation: z.ZodString;
|
|
14
|
+
}, z.core.$strict>;
|
|
@@ -2,7 +2,7 @@ export type TranslationJson = null | boolean | number | string | TranslationJson
|
|
|
2
2
|
[key: string]: TranslationJson;
|
|
3
3
|
};
|
|
4
4
|
export type TranslationConfiguration = Readonly<Record<string, TranslationJson>>;
|
|
5
|
-
export type TranslationErrorCode = 'disabled' | 'not-configured' | 'engine-unavailable' | 'authentication-failed' | 'invalid-request' | 'limit-exceeded' | 'timeout' | 'cancelled' | 'stale-result' | 'translation-failed' | 'invalid-model-output' | 'incomplete-model-output' | 'empty-model-output'
|
|
5
|
+
export type TranslationErrorCode = 'disabled' | 'not-configured' | 'engine-unavailable' | 'authentication-failed' | 'invalid-request' | 'limit-exceeded' | 'timeout' | 'cancelled' | 'stale-result' | 'translation-failed' | 'invalid-model-output' | 'incomplete-model-output' | 'empty-model-output';
|
|
6
6
|
export declare class TranslationError extends Error {
|
|
7
7
|
readonly code: TranslationErrorCode;
|
|
8
8
|
constructor(code: TranslationErrorCode, message: string, options?: ErrorOptions);
|
|
@@ -39,12 +39,22 @@ export interface TranslationEngineRequest extends Omit<TranslationRequest, 'sign
|
|
|
39
39
|
timeoutMs: number;
|
|
40
40
|
maxOutputCharacters: number;
|
|
41
41
|
}
|
|
42
|
+
/** The source used for this result, distinct from the user's unchanged selection.
|
|
43
|
+
* Certainty is the engine's assessment, not proof that a completion is original. */
|
|
44
|
+
export interface TranslationInterpretation {
|
|
45
|
+
text: string;
|
|
46
|
+
kind: 'correction' | 'completion';
|
|
47
|
+
certainty: 'confident' | 'tentative';
|
|
48
|
+
explanation: string;
|
|
49
|
+
}
|
|
42
50
|
export interface TranslationOutput {
|
|
43
51
|
text: string;
|
|
44
52
|
status: 'complete' | 'partial';
|
|
45
53
|
/** Stable warning identifiers suitable for localization by the consuming UI. */
|
|
46
54
|
warnings: string[];
|
|
47
55
|
detectedSourceLanguage?: string;
|
|
56
|
+
/** Omit when translating the selection as-is. Never write this back to the PDF. */
|
|
57
|
+
interpretation?: TranslationInterpretation;
|
|
48
58
|
/** Lexical detail is present only when an engine can validate it. */
|
|
49
59
|
detail?: {
|
|
50
60
|
kind: 'word' | 'phrase';
|
|
@@ -63,7 +73,7 @@ export interface TranslationEngineDescriptor {
|
|
|
63
73
|
id: string;
|
|
64
74
|
name: string;
|
|
65
75
|
version: string;
|
|
66
|
-
execution: '
|
|
76
|
+
execution: import('../modules/engines/types.js').EngineDescriptor['execution'];
|
|
67
77
|
maxInputCharacters?: number;
|
|
68
78
|
configurationFields?: import('../modules/engines/types.js').EngineConfigurationField[];
|
|
69
79
|
}
|