@dsh-libing/knowledge-base 0.1.0 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +12 -2
- package/lib/agent-access.d.ts +23 -0
- package/lib/agent-access.js +28 -0
- package/lib/chunks.js +27 -5
- package/lib/client-types/client/JobDetails.d.ts +16 -0
- package/lib/client-types/client/JobsPanel.d.ts +6 -1
- package/lib/client-types/client/QualityReport.d.ts +3 -2
- package/lib/client-types/client/api.d.ts +83 -52
- package/lib/client-types/client/locales.d.ts +2 -0
- package/lib/client-types/client/useKnowledge.d.ts +30 -15
- package/lib/client-types/contract.d.ts +83 -52
- package/lib/client-types/data.d.ts +11 -0
- package/lib/client-types/settings.d.ts +4 -0
- package/lib/client-types/shared.d.ts +17 -0
- package/lib/client.js +40 -40
- package/lib/config.js +9 -1
- package/lib/contract.d.ts +33 -2
- package/lib/contract.js +2 -1
- package/lib/data.d.ts +11 -0
- package/lib/data.js +2 -0
- package/lib/document/convert.js +52 -16
- package/lib/document/pdf-layout.js +45 -10
- package/lib/document/pdf-session.js +10 -4
- package/lib/document/pdf-vision.js +8 -3
- package/lib/document/pdf-worker.js +76 -21
- package/lib/document/pdf.js +39 -9
- package/lib/document/zip.js +14 -6
- package/lib/figures/image-client.d.ts +5 -0
- package/lib/figures/image-client.js +63 -0
- package/lib/figures/image-gate.d.ts +12 -0
- package/lib/figures/image-gate.js +39 -0
- package/lib/figures/image-protocol.d.ts +19 -0
- package/lib/figures/image-protocol.js +9 -0
- package/lib/figures/image-worker.d.ts +1 -0
- package/lib/figures/image-worker.js +69 -0
- package/lib/figures/process.d.ts +1 -1
- package/lib/figures/process.js +19 -38
- package/lib/index.js +6 -2
- package/lib/indexes.js +17 -6
- package/lib/job-storage.d.ts +264 -0
- package/lib/job-storage.js +118 -0
- package/lib/jobs.d.ts +782 -1489
- package/lib/jobs.js +64 -31
- package/lib/models.d.ts +1 -1
- package/lib/models.js +15 -9
- package/lib/retrieval.js +44 -13
- package/lib/revision-cache.d.ts +24 -0
- package/lib/revision-cache.js +100 -0
- package/lib/routes.js +66 -20
- package/lib/runtime.d.ts +4 -0
- package/lib/runtime.js +15 -4
- package/lib/settings.d.ts +4 -0
- package/lib/settings.js +5 -2
- package/lib/shared.d.ts +17 -0
- package/lib/shared.js +3 -0
- package/lib/skills.js +6 -3
- package/lib/store.d.ts +101 -5
- package/lib/store.js +78 -29
- package/lib/tools.js +12 -4
- package/lib/util.js +2 -1
- package/package.json +1 -1
package/lib/config.js
CHANGED
|
@@ -10,4 +10,12 @@ export function resolveConfig(raw = {}) {
|
|
|
10
10
|
return settingsSchema.parse({ ...raw, dataRoot });
|
|
11
11
|
}
|
|
12
12
|
const defaults = resolveConfig();
|
|
13
|
-
export const Config = Schema.object(Object.fromEntries(Object.entries(defaults).filter(([key]) => !['skipImages', 'acceptPartial'].includes(key)).map(([key, value]) => [key,
|
|
13
|
+
export const Config = Schema.object(Object.fromEntries(Object.entries(defaults).filter(([key]) => !['skipImages', 'acceptPartial'].includes(key)).map(([key, value]) => [key, (() => {
|
|
14
|
+
if (typeof value === 'number') {
|
|
15
|
+
return Schema.number().min(1).default(value);
|
|
16
|
+
}
|
|
17
|
+
if (typeof value === 'string') {
|
|
18
|
+
return Schema.string().default(value);
|
|
19
|
+
}
|
|
20
|
+
return Schema.boolean().default(value);
|
|
21
|
+
})()])));
|
package/lib/contract.d.ts
CHANGED
|
@@ -119,8 +119,12 @@ export declare const publicJobSchema: z.ZodObject<{
|
|
|
119
119
|
modelTimeoutMs: z.ZodDefault<z.ZodNumber>;
|
|
120
120
|
modelAttempts: z.ZodDefault<z.ZodNumber>;
|
|
121
121
|
visionConcurrency: z.ZodDefault<z.ZodNumber>;
|
|
122
|
+
maxModelCallsLimit: z.ZodDefault<z.ZodNumber>;
|
|
123
|
+
maxModelTokensLimit: z.ZodDefault<z.ZodNumber>;
|
|
124
|
+
uploadRetentionDays: z.ZodDefault<z.ZodNumber>;
|
|
122
125
|
maxModelCalls: z.ZodDefault<z.ZodNumber>;
|
|
123
126
|
maxModelTokens: z.ZodDefault<z.ZodNumber>;
|
|
127
|
+
imageTimeoutMs: z.ZodDefault<z.ZodNumber>;
|
|
124
128
|
visionMaxEdge: z.ZodDefault<z.ZodNumber>;
|
|
125
129
|
visionTileSize: z.ZodDefault<z.ZodNumber>;
|
|
126
130
|
visionMaxTiles: z.ZodDefault<z.ZodNumber>;
|
|
@@ -189,6 +193,13 @@ export declare const publicJobSchema: z.ZodObject<{
|
|
|
189
193
|
chunkId: z.ZodString;
|
|
190
194
|
}, z.core.$strip>>;
|
|
191
195
|
}, z.core.$strip>;
|
|
196
|
+
pendingCalls: z.ZodDefault<z.ZodArray<z.ZodNumber>>;
|
|
197
|
+
legacyUsageCount: z.ZodDefault<z.ZodNumber>;
|
|
198
|
+
legacyCallCount: z.ZodDefault<z.ZodNumber>;
|
|
199
|
+
reportSummary: z.ZodOptional<z.ZodObject<{
|
|
200
|
+
issues: z.ZodNumber;
|
|
201
|
+
blocking: z.ZodNumber;
|
|
202
|
+
}, z.core.$strip>>;
|
|
192
203
|
failures: z.ZodArray<z.ZodObject<{
|
|
193
204
|
at: z.ZodString;
|
|
194
205
|
stage: z.ZodEnum<{
|
|
@@ -437,8 +448,12 @@ export declare const stateSchema: z.ZodObject<{
|
|
|
437
448
|
modelTimeoutMs: z.ZodDefault<z.ZodNumber>;
|
|
438
449
|
modelAttempts: z.ZodDefault<z.ZodNumber>;
|
|
439
450
|
visionConcurrency: z.ZodDefault<z.ZodNumber>;
|
|
451
|
+
maxModelCallsLimit: z.ZodDefault<z.ZodNumber>;
|
|
452
|
+
maxModelTokensLimit: z.ZodDefault<z.ZodNumber>;
|
|
453
|
+
uploadRetentionDays: z.ZodDefault<z.ZodNumber>;
|
|
440
454
|
maxModelCalls: z.ZodDefault<z.ZodNumber>;
|
|
441
455
|
maxModelTokens: z.ZodDefault<z.ZodNumber>;
|
|
456
|
+
imageTimeoutMs: z.ZodDefault<z.ZodNumber>;
|
|
442
457
|
visionMaxEdge: z.ZodDefault<z.ZodNumber>;
|
|
443
458
|
visionTileSize: z.ZodDefault<z.ZodNumber>;
|
|
444
459
|
visionMaxTiles: z.ZodDefault<z.ZodNumber>;
|
|
@@ -507,6 +522,13 @@ export declare const stateSchema: z.ZodObject<{
|
|
|
507
522
|
chunkId: z.ZodString;
|
|
508
523
|
}, z.core.$strip>>;
|
|
509
524
|
}, z.core.$strip>;
|
|
525
|
+
pendingCalls: z.ZodDefault<z.ZodArray<z.ZodNumber>>;
|
|
526
|
+
legacyUsageCount: z.ZodDefault<z.ZodNumber>;
|
|
527
|
+
legacyCallCount: z.ZodDefault<z.ZodNumber>;
|
|
528
|
+
reportSummary: z.ZodOptional<z.ZodObject<{
|
|
529
|
+
issues: z.ZodNumber;
|
|
530
|
+
blocking: z.ZodNumber;
|
|
531
|
+
}, z.core.$strip>>;
|
|
510
532
|
failures: z.ZodArray<z.ZodObject<{
|
|
511
533
|
at: z.ZodString;
|
|
512
534
|
stage: z.ZodEnum<{
|
|
@@ -557,6 +579,9 @@ export declare const stateSchema: z.ZodObject<{
|
|
|
557
579
|
outputPath: z.ZodNullable<z.ZodString>;
|
|
558
580
|
reused: z.ZodOptional<z.ZodBoolean>;
|
|
559
581
|
}, z.core.$strip>>;
|
|
582
|
+
jobTotal: z.ZodDefault<z.ZodNumber>;
|
|
583
|
+
jobOffset: z.ZodDefault<z.ZodNumber>;
|
|
584
|
+
activeJobs: z.ZodDefault<z.ZodNumber>;
|
|
560
585
|
dataRoot: z.ZodString;
|
|
561
586
|
config: z.ZodObject<{
|
|
562
587
|
chunkTargetChars: z.ZodNumber;
|
|
@@ -669,8 +694,8 @@ export declare const treeDocumentSchema: z.ZodObject<{
|
|
|
669
694
|
reason: z.ZodOptional<z.ZodString>;
|
|
670
695
|
}, z.core.$strip>>>;
|
|
671
696
|
chunks: z.ZodArray<z.ZodObject<{
|
|
672
|
-
summary: z.ZodDefault<z.ZodString>;
|
|
673
697
|
path: z.ZodString;
|
|
698
|
+
summary: z.ZodDefault<z.ZodString>;
|
|
674
699
|
title: z.ZodString;
|
|
675
700
|
headingPath: z.ZodArray<z.ZodString>;
|
|
676
701
|
oversizedReason: z.ZodOptional<z.ZodString>;
|
|
@@ -972,8 +997,8 @@ export declare const treeSchema: z.ZodObject<{
|
|
|
972
997
|
reason: z.ZodOptional<z.ZodString>;
|
|
973
998
|
}, z.core.$strip>>>;
|
|
974
999
|
chunks: z.ZodArray<z.ZodObject<{
|
|
975
|
-
summary: z.ZodDefault<z.ZodString>;
|
|
976
1000
|
path: z.ZodString;
|
|
1001
|
+
summary: z.ZodDefault<z.ZodString>;
|
|
977
1002
|
title: z.ZodString;
|
|
978
1003
|
headingPath: z.ZodArray<z.ZodString>;
|
|
979
1004
|
oversizedReason: z.ZodOptional<z.ZodString>;
|
|
@@ -1195,3 +1220,9 @@ export declare const uploadResultSchema: z.ZodObject<{
|
|
|
1195
1220
|
filename: z.ZodString;
|
|
1196
1221
|
bytes: z.ZodNumber;
|
|
1197
1222
|
}, z.core.$strip>;
|
|
1223
|
+
export declare const jobUsagePageSchema: z.ZodObject<{
|
|
1224
|
+
records: z.ZodArray<z.ZodJSONSchema>;
|
|
1225
|
+
offset: z.ZodNumber;
|
|
1226
|
+
total: z.ZodNumber;
|
|
1227
|
+
next: z.ZodNullable<z.ZodNumber>;
|
|
1228
|
+
}, z.core.$strip>;
|
package/lib/contract.js
CHANGED
|
@@ -4,7 +4,7 @@ import { jobSchema, registrySchema, librarySchema, publishedDocumentSchema, repo
|
|
|
4
4
|
export const publicJobSchema = jobSchema.omit({ schema: true, overlays: true }).extend({ outputPath: z.string().nullable(), reused: z.boolean().optional() });
|
|
5
5
|
export const locationSchema = z.object({ kbPath: z.string(), revision: z.string().nullable(), wikiPath: z.string().nullable(), indexPath: z.string().nullable() });
|
|
6
6
|
export const libraryViewSchema = librarySchema.extend({ location: locationSchema, summary: z.object({ revision: z.string().nullable(), documentCount: z.number(), chunkCount: z.number(), coverageIncomplete: z.boolean() }) });
|
|
7
|
-
export const stateSchema = z.object({ hostVersion: z.string(), kbs: z.array(libraryViewSchema), deletedKbs: z.array(librarySchema), activities: registrySchema.shape.activities, jobs: z.array(publicJobSchema), dataRoot: z.string(), config: z.object({ chunkTargetChars: z.number(), visionProvider: z.string(), visionModel: z.string(), maxInputBytes: z.number() }) });
|
|
7
|
+
export const stateSchema = z.object({ hostVersion: z.string(), kbs: z.array(libraryViewSchema), deletedKbs: z.array(librarySchema), activities: registrySchema.shape.activities, jobs: z.array(publicJobSchema), jobTotal: z.number().int().nonnegative().default(0), jobOffset: z.number().int().nonnegative().default(0), activeJobs: z.number().int().nonnegative().default(0), dataRoot: z.string(), config: z.object({ chunkTargetChars: z.number(), visionProvider: z.string(), visionModel: z.string(), maxInputBytes: z.number() }) });
|
|
8
8
|
export const treeDocumentSchema = publishedDocumentSchema.extend({ chunks: z.array(publishedDocumentSchema.shape.chunks.element.omit({ segments: true, anchors: true })), figures: z.array(publishedDocumentSchema.shape.figures.element.extend({ expectedHash: z.string().optional() })) });
|
|
9
9
|
export const treeSchema = z.object({ kbId: z.string(), revision: z.string(), location: locationSchema.optional(), previewJobId: z.string().optional(), coverageIncomplete: z.boolean(), report: reportSchema, documents: z.array(treeDocumentSchema) });
|
|
10
10
|
export const readSchema = z.object({ kbId: z.string(), chunkId: z.string(), revision: z.string(), text: z.string(), complete: z.boolean(), truncated: z.boolean().optional(), previewJobId: z.string().optional(),
|
|
@@ -16,3 +16,4 @@ export const modelsSchema = z.object({ providers: z.array(z.object({ id: z.strin
|
|
|
16
16
|
export const schemaResultSchema = z.object({ content: z.string(), hash: z.string() });
|
|
17
17
|
export const inspectSchema = z.object({ sourceFormat: z.literal('pdf'), pageCount: z.number(), outlineCount: z.number(), samples: z.array(z.object({ page: z.number(), chars: z.number(), hasGraphics: z.boolean(), possibleOcr: z.boolean(), hasStructure: z.boolean() })), possibleOcr: z.boolean(), note: z.string() });
|
|
18
18
|
export const uploadResultSchema = z.object({ receiptId: z.string(), filename: z.string(), bytes: z.number() });
|
|
19
|
+
export const jobUsagePageSchema = z.object({ records: z.array(z.json()), offset: z.number().int().nonnegative(), total: z.number().int().nonnegative(), next: z.number().int().nonnegative().nullable() });
|
package/lib/data.d.ts
CHANGED
|
@@ -1528,8 +1528,12 @@ export declare const jobSchema: z.ZodObject<{
|
|
|
1528
1528
|
modelTimeoutMs: z.ZodDefault<z.ZodNumber>;
|
|
1529
1529
|
modelAttempts: z.ZodDefault<z.ZodNumber>;
|
|
1530
1530
|
visionConcurrency: z.ZodDefault<z.ZodNumber>;
|
|
1531
|
+
maxModelCallsLimit: z.ZodDefault<z.ZodNumber>;
|
|
1532
|
+
maxModelTokensLimit: z.ZodDefault<z.ZodNumber>;
|
|
1533
|
+
uploadRetentionDays: z.ZodDefault<z.ZodNumber>;
|
|
1531
1534
|
maxModelCalls: z.ZodDefault<z.ZodNumber>;
|
|
1532
1535
|
maxModelTokens: z.ZodDefault<z.ZodNumber>;
|
|
1536
|
+
imageTimeoutMs: z.ZodDefault<z.ZodNumber>;
|
|
1533
1537
|
visionMaxEdge: z.ZodDefault<z.ZodNumber>;
|
|
1534
1538
|
visionTileSize: z.ZodDefault<z.ZodNumber>;
|
|
1535
1539
|
visionMaxTiles: z.ZodDefault<z.ZodNumber>;
|
|
@@ -1599,6 +1603,13 @@ export declare const jobSchema: z.ZodObject<{
|
|
|
1599
1603
|
chunkId: z.ZodString;
|
|
1600
1604
|
}, z.core.$strip>>;
|
|
1601
1605
|
}, z.core.$strip>;
|
|
1606
|
+
pendingCalls: z.ZodDefault<z.ZodArray<z.ZodNumber>>;
|
|
1607
|
+
legacyUsageCount: z.ZodDefault<z.ZodNumber>;
|
|
1608
|
+
legacyCallCount: z.ZodDefault<z.ZodNumber>;
|
|
1609
|
+
reportSummary: z.ZodOptional<z.ZodObject<{
|
|
1610
|
+
issues: z.ZodNumber;
|
|
1611
|
+
blocking: z.ZodNumber;
|
|
1612
|
+
}, z.core.$strip>>;
|
|
1602
1613
|
usage: z.ZodObject<{
|
|
1603
1614
|
calls: z.ZodNumber;
|
|
1604
1615
|
tokens: z.ZodNumber;
|
package/lib/data.js
CHANGED
|
@@ -69,6 +69,8 @@ export const jobSchema = z.object({ jobId: idSchema, kbId: idSchema, docId: idSc
|
|
|
69
69
|
progress: z.object({ totalPages: z.number().optional(), parsedPages: z.number().optional(), ocrPages: z.number().optional(), tables: z.number().optional(), blocks: z.number().optional(), currentPage: z.number().optional(),
|
|
70
70
|
totalFigures: z.number().optional(), completedFigures: z.number().optional(), inflightFigures: z.array(z.object({ occurrenceId: z.string(), caption: z.string() })).optional(),
|
|
71
71
|
totalChunks: z.number().optional(), indexedChunks: z.number().optional(), currentChunk: z.object({ title: z.string(), headingPath: z.array(z.string()), chunkId: z.string() }).optional() }),
|
|
72
|
+
pendingCalls: z.array(z.number().int().positive()).max(16).default([]), legacyUsageCount: z.number().int().nonnegative().default(0), legacyCallCount: z.number().int().nonnegative().default(0),
|
|
73
|
+
reportSummary: z.object({ issues: z.number().int().nonnegative(), blocking: z.number().int().nonnegative() }).optional(),
|
|
72
74
|
usage: usageSchema, failures: z.array(z.object({ at: z.string(), stage: jobStateSchema, error: z.string() })), events: z.array(z.object({ at: z.string(), state: jobStateSchema })).optional(),
|
|
73
75
|
figureRefresh: z.object({ occurrenceId: z.string(), token: z.string() }).nullable().optional(), visionCapability: z.string().optional(), parserVersion: z.string().optional(),
|
|
74
76
|
pageStart: z.number().optional(), pageEnd: z.number().optional(), pdf: z.object({ pageCount: z.number(), selectedPages: rangeSchema, outlineCount: z.number().optional() }).optional(),
|
package/lib/document/convert.js
CHANGED
|
@@ -12,7 +12,15 @@ import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
|
|
12
12
|
import { tmpdir } from 'node:os';
|
|
13
13
|
const parser = new XMLParser({ ignoreAttributes: false, removeNSPrefix: true, parseTagValue: false, trimValues: false });
|
|
14
14
|
const xmlSchema = z.lazy(() => z.union([z.string(), z.number(), z.boolean(), z.null(), z.array(xmlSchema), z.record(z.string(), xmlSchema)]));
|
|
15
|
-
const list = (value) =>
|
|
15
|
+
const list = (value) => {
|
|
16
|
+
if (value === undefined) {
|
|
17
|
+
return [];
|
|
18
|
+
}
|
|
19
|
+
if (Array.isArray(value)) {
|
|
20
|
+
return value;
|
|
21
|
+
}
|
|
22
|
+
return [value];
|
|
23
|
+
};
|
|
16
24
|
const object = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
|
17
25
|
function child(value, ...keys) {
|
|
18
26
|
for (const key of keys)
|
|
@@ -43,8 +51,13 @@ function wordText(value) {
|
|
|
43
51
|
}
|
|
44
52
|
const normalize = (value) => value.replace(/\s/gu, '');
|
|
45
53
|
function audit(files) {
|
|
46
|
-
const doc = parseXml(required(files['word/document.xml']))
|
|
47
|
-
const
|
|
54
|
+
const doc = parseXml(required(files['word/document.xml']));
|
|
55
|
+
const styles = files['word/styles.xml'] ? parseXml(files['word/styles.xml']) : {};
|
|
56
|
+
const warnings = [];
|
|
57
|
+
const styleLevels = {};
|
|
58
|
+
const headingLevels = {};
|
|
59
|
+
const outlineLevels = {};
|
|
60
|
+
const anchors = new Set();
|
|
48
61
|
const styleRows = list(child(styles, 'styles', 'style'));
|
|
49
62
|
const resolveLevel = (styleId, seen = new Set()) => {
|
|
50
63
|
if (seen.has(styleId) || !styleId)
|
|
@@ -53,19 +66,33 @@ function audit(files) {
|
|
|
53
66
|
const row = styleRows.find(v => xmlString(child(v, '@_styleId')) === styleId);
|
|
54
67
|
if (!row)
|
|
55
68
|
return;
|
|
56
|
-
const explicit = child(row, 'pPr', 'outlineLvl', '@_val')
|
|
57
|
-
|
|
69
|
+
const explicit = child(row, 'pPr', 'outlineLvl', '@_val');
|
|
70
|
+
const name = xmlString(child(row, 'name', '@_val')) || styleId;
|
|
71
|
+
const common = /^(?:heading|标题)\s*([1-9])$/iu.exec(name);
|
|
72
|
+
if (explicit !== undefined && Number(explicit) < 9) {
|
|
73
|
+
return Number(explicit) + 1;
|
|
74
|
+
}
|
|
75
|
+
if (common) {
|
|
76
|
+
return Number(common[1]);
|
|
77
|
+
}
|
|
78
|
+
return resolveLevel(xmlString(child(row, 'basedOn', '@_val')), seen);
|
|
58
79
|
};
|
|
59
80
|
for (const row of styleRows) {
|
|
60
|
-
const styleId = xmlString(child(row, '@_styleId'))
|
|
81
|
+
const styleId = xmlString(child(row, '@_styleId'));
|
|
82
|
+
const level = resolveLevel(styleId);
|
|
61
83
|
if (level)
|
|
62
84
|
styleLevels[styleId] = level;
|
|
63
85
|
}
|
|
64
|
-
let expectedFigures = 0
|
|
86
|
+
let expectedFigures = 0;
|
|
87
|
+
let expectedTables = 0;
|
|
88
|
+
let textBoxes = 0;
|
|
89
|
+
let footnoteRefs = 0;
|
|
65
90
|
const expectedHeadings = [];
|
|
66
91
|
visit(doc, (key, value) => {
|
|
67
92
|
if (key === 'p') {
|
|
68
|
-
const outline = child(value, 'pPr', 'outlineLvl', '@_val')
|
|
93
|
+
const outline = child(value, 'pPr', 'outlineLvl', '@_val');
|
|
94
|
+
const style = xmlString(child(value, 'pPr', 'pStyle', '@_val'));
|
|
95
|
+
const level = outline !== undefined && Number(outline) < 9 ? Number(outline) + 1 : styleLevels[style];
|
|
69
96
|
if (outline !== undefined && Number(outline) < 9)
|
|
70
97
|
outlineLevels[normalize(wordText(value))] = Number(outline) + 1;
|
|
71
98
|
if (level && !/^TOC/iu.test(style)) {
|
|
@@ -119,7 +146,8 @@ function safeLink(value) { return /^(?:https?:|mailto:|#)/i.test(value || '') ?
|
|
|
119
146
|
function inline(node, internalRefs) {
|
|
120
147
|
if ('value' in node)
|
|
121
148
|
return node.value.replace(/([\\`*[\]<>])/g, '\\$1');
|
|
122
|
-
const t = tag(node)
|
|
149
|
+
const t = tag(node);
|
|
150
|
+
const body = children(node).map(n => inline(n, internalRefs)).join('');
|
|
123
151
|
if (['script', 'style', 'iframe', 'object'].includes(t))
|
|
124
152
|
return '';
|
|
125
153
|
if (t === 'img')
|
|
@@ -159,8 +187,10 @@ function tableHtml(rows) {
|
|
|
159
187
|
}
|
|
160
188
|
export async function convertDocx(buffer, destination, config, signal) {
|
|
161
189
|
checkAbort(signal);
|
|
162
|
-
const files = unpackDocx(buffer, config)
|
|
163
|
-
const
|
|
190
|
+
const files = unpackDocx(buffer, config);
|
|
191
|
+
const originalAudit = audit(files);
|
|
192
|
+
const assets = [];
|
|
193
|
+
const warnings = [...originalAudit.warnings];
|
|
164
194
|
const mimeExtension = { 'image/png': 'png', 'image/jpeg': 'jpg', 'image/gif': 'gif', 'image/webp': 'webp', 'image/bmp': 'bmp', 'image/tiff': 'tif', 'image/x-emf': 'emf', 'image/x-wmf': 'wmf', 'image/svg+xml': 'svg' };
|
|
165
195
|
const addAsset = async (data, mime, origin) => {
|
|
166
196
|
const assetId = `a-${hash(data).slice(0, 20)}`;
|
|
@@ -174,7 +204,8 @@ export async function convertDocx(buffer, destination, config, signal) {
|
|
|
174
204
|
};
|
|
175
205
|
const rawMedia = Object.entries(files).filter(([name]) => name.startsWith('word/media/'));
|
|
176
206
|
for (const [name, bytes] of rawMedia) {
|
|
177
|
-
const ext = (name.split('.').at(-1) || '').toLowerCase()
|
|
207
|
+
const ext = (name.split('.').at(-1) || '').toLowerCase();
|
|
208
|
+
const mime = Object.entries(mimeExtension).find(([, x]) => x === ext || ext === 'jpeg' && x === 'jpg')?.[0] || 'application/octet-stream';
|
|
178
209
|
await addAsset(Buffer.from(bytes), mime, name);
|
|
179
210
|
}
|
|
180
211
|
const result = await mammoth.convertToHtml({ buffer }, {
|
|
@@ -208,11 +239,14 @@ export async function convertDocx(buffer, destination, config, signal) {
|
|
|
208
239
|
// Unmapped styles lose formatting, not paragraph text; heading coverage is audited separately.
|
|
209
240
|
blocking: !/^Unrecognised (?:paragraph|run) style:/i.test(m.message) && /image|unrecognised|unsupported/i.test(m.message),
|
|
210
241
|
})));
|
|
211
|
-
const fragment = parseFragment(result.value)
|
|
242
|
+
const fragment = parseFragment(result.value);
|
|
243
|
+
const blocks = [];
|
|
244
|
+
const internalRefs = [];
|
|
212
245
|
let figureCount = 0;
|
|
213
246
|
const add = (value) => { blocks.push(makeBlock({ blockId: `b-${String(blocks.length + 1).padStart(6, '0')}`, ...value })); return required(blocks.at(-1)); };
|
|
214
247
|
const addInline = (node, type = 'paragraph', prefix = '') => {
|
|
215
|
-
const body = inline(node, internalRefs)
|
|
248
|
+
const body = inline(node, internalRefs);
|
|
249
|
+
const pieces = body.split(String.fromCharCode(1)).filter(Boolean);
|
|
216
250
|
for (const piece of pieces) {
|
|
217
251
|
if (piece.startsWith('asset:')) {
|
|
218
252
|
figureCount++;
|
|
@@ -227,7 +261,8 @@ export async function convertDocx(buffer, destination, config, signal) {
|
|
|
227
261
|
if (/^h[1-9]$/.test(t))
|
|
228
262
|
add({ type: 'heading', level: Number(t[1]), text: htmlText(node).trim(), anchor: attr(node, 'id') || null });
|
|
229
263
|
else if (t === 'table') {
|
|
230
|
-
const rows = tableCells(node)
|
|
264
|
+
const rows = tableCells(node);
|
|
265
|
+
const table = add({ type: 'table', rows, text: rows.map(row => row.map(c => c.text).join(' | ')).join('\n'), markdown: tableHtml(rows) });
|
|
231
266
|
const scan = (n) => {
|
|
232
267
|
if (tag(n) === 'img') {
|
|
233
268
|
const assetId = (attr(n, 'src') || '').replace(/^asset:/, '');
|
|
@@ -241,7 +276,8 @@ export async function convertDocx(buffer, destination, config, signal) {
|
|
|
241
276
|
scan(node);
|
|
242
277
|
}
|
|
243
278
|
else if (t === 'p') {
|
|
244
|
-
const t = htmlText(node).trim()
|
|
279
|
+
const t = htmlText(node).trim();
|
|
280
|
+
const previous = blocks.at(-1);
|
|
245
281
|
if (/^(?:图|Figure)\s*[A-Za-z]?\d[\d.\-–]*/i.test(t) && previous?.type === 'figure') {
|
|
246
282
|
previous.caption = t;
|
|
247
283
|
add({ type: 'caption', text: t, figureRef: previous.occurrenceId });
|
|
@@ -19,7 +19,8 @@ export function lines(spans) {
|
|
|
19
19
|
}
|
|
20
20
|
return rows.sort((a, b) => a.y - b.y).map(r => {
|
|
21
21
|
const items = r.spans.sort((a, b) => readingBox(a)[0] - readingBox(b)[0]);
|
|
22
|
-
let text = ''
|
|
22
|
+
let text = '';
|
|
23
|
+
let last;
|
|
23
24
|
for (const s of items) {
|
|
24
25
|
if (last && readingBox(s)[0] - readingBox(last)[2] > Math.max(1, s.fontSize * .18) && !/\s$/.test(text))
|
|
25
26
|
text += ' ';
|
|
@@ -83,16 +84,23 @@ function geometricTables(page, spans, occupied) {
|
|
|
83
84
|
group.push(required(rows[++i]));
|
|
84
85
|
if (group.length < 2)
|
|
85
86
|
continue;
|
|
86
|
-
const ids = new Set()
|
|
87
|
+
const ids = new Set();
|
|
88
|
+
const cells = group.map(r => r.cells.map(b => { const ss = spans.filter(s => { const x = (s.bbox[0] + s.bbox[2]) / 2; const y = (s.bbox[1] + s.bbox[3]) / 2; return x > b[0] && x < b[2] && y > b[1] && y < b[3]; }); ss.forEach(s => ids.add(s.id)); return { text: textOf(ss), spanIds: ss.map(s => s.id), bbox: b, colspan: 1, rowspan: 1 }; }));
|
|
87
89
|
if (cells.some(r => r.every(c => !c.text)))
|
|
88
90
|
continue;
|
|
89
|
-
const bbox = required(union(group.flatMap(r => r.cells)))
|
|
91
|
+
const bbox = required(union(group.flatMap(r => r.cells)));
|
|
92
|
+
const identified = /field|offset|bits|name|description|字段|偏移|名称|参数|描述|位范围/i.test(required(cells[0]).map(c => c.text).join(' '));
|
|
90
93
|
tables.push({ rows: cells, bbox, ids, identified });
|
|
91
94
|
}
|
|
92
95
|
return tables;
|
|
93
96
|
}
|
|
94
97
|
export function nativeLayout(page, info) {
|
|
95
|
-
const byMarked = new Map()
|
|
98
|
+
const byMarked = new Map();
|
|
99
|
+
const used = new Set();
|
|
100
|
+
const blocks = [];
|
|
101
|
+
const figures = [];
|
|
102
|
+
const warnings = (page.warnings || []).map(message => ({ code: 'pdf-engine-warning', page: page.page, message: `PDF 第 ${page.page} 页解析器警告:${message}`, blocking: true }));
|
|
103
|
+
const excluded = page.spans.filter(s => s.artifact).map(s => ({ kind: 'tagged-artifact', text: s.text, bbox: s.bbox }));
|
|
96
104
|
const textSpans = page.spans.filter(s => !s.artifact);
|
|
97
105
|
for (const s of textSpans) {
|
|
98
106
|
if (!byMarked.has(s.mcid))
|
|
@@ -128,7 +136,8 @@ export function nativeLayout(page, info) {
|
|
|
128
136
|
if (!rows.length)
|
|
129
137
|
return;
|
|
130
138
|
const irregular = rows.some(r => r.length !== required(rows[0]).length);
|
|
131
|
-
const textBox = required(union(spans.map(s => s.bbox)))
|
|
139
|
+
const textBox = required(union(spans.map(s => s.bbox)));
|
|
140
|
+
const nearby = page.drawings.filter(d => d.kind === 'path' && d.bbox[1] >= textBox[1] - 20 && d.bbox[3] <= textBox[3] + 20 && intersects(d.bbox, boundedBox(textBox, page, 10))).map(d => d.bbox);
|
|
132
141
|
const tableBox = boundedBox(required(union([textBox, ...nearby])), page, 3);
|
|
133
142
|
add('table', spans, { source: source(tableBox), rows, markdown: tableMarkdown(rows), needsReview: irregular, caption: '', layoutBasis: 'structure-tags', ...textBox[3] > page.height * .85 && !closedTableBottom(page, textBox, tableBox) ? { openEnd: true } : {} });
|
|
134
143
|
if (irregular)
|
|
@@ -136,7 +145,10 @@ export function nativeLayout(page, info) {
|
|
|
136
145
|
return;
|
|
137
146
|
}
|
|
138
147
|
if (node.role === 'Figure') {
|
|
139
|
-
const marked = ids(node)
|
|
148
|
+
const marked = ids(node);
|
|
149
|
+
const spans = get(node);
|
|
150
|
+
const boxes = [...page.drawings.filter(d => d.mcid !== null && marked.includes(d.mcid)).map(d => d.bbox), ...spans.map(s => s.bbox)];
|
|
151
|
+
const bbox = union(boxes);
|
|
140
152
|
if (!bbox)
|
|
141
153
|
return;
|
|
142
154
|
consume(spans);
|
|
@@ -148,7 +160,15 @@ export function nativeLayout(page, info) {
|
|
|
148
160
|
return;
|
|
149
161
|
}
|
|
150
162
|
if (['Code', 'Formula', 'Note'].includes(node.role || '')) {
|
|
151
|
-
add(
|
|
163
|
+
add((() => {
|
|
164
|
+
if (node.role === 'Code') {
|
|
165
|
+
return 'code';
|
|
166
|
+
}
|
|
167
|
+
if (node.role === 'Formula') {
|
|
168
|
+
return 'formula';
|
|
169
|
+
}
|
|
170
|
+
return 'note';
|
|
171
|
+
})(), get(node));
|
|
152
172
|
return;
|
|
153
173
|
}
|
|
154
174
|
if (node.role === 'P') {
|
|
@@ -194,8 +214,21 @@ export function nativeLayout(page, info) {
|
|
|
194
214
|
const match = info.outline.find(o => {
|
|
195
215
|
if (o.page !== page.page || normalize(o.title) !== normalize(b.text))
|
|
196
216
|
return false;
|
|
197
|
-
const candidates = blocks.filter(c => normalize(c.text) === normalize(o.title))
|
|
198
|
-
const
|
|
217
|
+
const candidates = blocks.filter(c => normalize(c.text) === normalize(o.title));
|
|
218
|
+
const t = page.viewportTransform;
|
|
219
|
+
const axis = page.rotation % 180 ? 0 : 1;
|
|
220
|
+
let target;
|
|
221
|
+
if (o.y !== null && Number.isFinite(o.y) && t) {
|
|
222
|
+
if (axis === 0) {
|
|
223
|
+
target = required(t[2]) * o.y + required(t[4]);
|
|
224
|
+
}
|
|
225
|
+
else {
|
|
226
|
+
target = required(t[3]) * o.y + required(t[5]);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
else {
|
|
230
|
+
target = null;
|
|
231
|
+
}
|
|
199
232
|
const score = (c) => target === null ? (c.type === 'heading' ? 0 : 100) + c.source.bbox[1] / 1000 : Math.abs(c.source.bbox[axis] - target);
|
|
200
233
|
return candidates.sort((a, b) => score(a) - score(b))[0] === b && (target === null || score(b) < 100);
|
|
201
234
|
});
|
|
@@ -225,7 +258,9 @@ export function nativeLayout(page, info) {
|
|
|
225
258
|
blocks.sort((a, b) => a.source.bbox[1] - b.source.bbox[1] || a.source.bbox[0] - b.source.bbox[0]);
|
|
226
259
|
const coverage = [...[...figures, ...blocks.filter(b => b.type === 'table')].map(b => boundedBox(b.source.bbox, page, 15)), ...excluded.map(e => boundedBox(e.bbox, page, 2))];
|
|
227
260
|
const unexplained = page.drawings.filter(d => d.bbox[2] - d.bbox[0] > 25 && d.bbox[3] - d.bbox[1] > 12 && d.bbox[1] > 65 && d.bbox[3] < page.height - 40 && !coverage.some(b => intersects(b, d.bbox)));
|
|
228
|
-
const raw = textSpans.map(s => s.text).join('')
|
|
261
|
+
const raw = textSpans.map(s => s.text).join('');
|
|
262
|
+
const garbled = textSpans.filter(s => (s.text.includes('\ufffd') || s.text.includes('\0')));
|
|
263
|
+
const ocr = (raw.length < 30 && page.drawings.length > 0) || garbled.length > textSpans.length * .3;
|
|
229
264
|
if (garbled.length)
|
|
230
265
|
warnings.push({ code: 'pdf-garbled-text', page: page.page, message: `PDF 第 ${page.page} 页存在缺字或乱码,需要核对识别区域`, blocking: true });
|
|
231
266
|
const multiColumn = lines(textSpans.filter(s => !coverage.some(b => intersects(s.bbox, b)))).filter(l => l.spans.some((s, i) => i && s.bbox[0] - required(l.spans[i - 1]).bbox[2] > page.width * .13)).length >= 3;
|
|
@@ -24,7 +24,8 @@ export class PdfSession {
|
|
|
24
24
|
this.signal = signal;
|
|
25
25
|
checkAbort(signal);
|
|
26
26
|
// Production uses the shipped JS worker. Source tests use the same TS through the installed tsx loader.
|
|
27
|
-
const js = new URL('./pdf-worker.js', import.meta.url)
|
|
27
|
+
const js = new URL('./pdf-worker.js', import.meta.url);
|
|
28
|
+
const source = !existsSync(fileURLToPath(js));
|
|
28
29
|
this.worker = new Worker(source ? new URL('./pdf-worker.ts', import.meta.url) : js, { workerData: { file, config: { maxInputBytes: config.maxInputBytes, pdfMaxPages: config.pdfMaxPages, pdfMaxPagePixels: config.pdfMaxPagePixels } }, resourceLimits: { maxOldGenerationSizeMb: 512 }, execArgv: source ? ['--import', 'tsx/esm'] : [] });
|
|
29
30
|
const open = deferred();
|
|
30
31
|
this.ready = open.promise;
|
|
@@ -85,7 +86,8 @@ export class PdfSession {
|
|
|
85
86
|
throw this.error;
|
|
86
87
|
if (this.closed)
|
|
87
88
|
throw new Error('PDF 解析进程已关闭');
|
|
88
|
-
const id = ++this.seq
|
|
89
|
+
const id = ++this.seq;
|
|
90
|
+
const value = await new Promise((resolve, reject) => { const timer = setTimeout(() => this.fail(new Error(`PDF 第 ${page} 页处理超时`)), this.config.pdfPageTimeoutMs); this.pending.set(id, { resolve, reject, timer }); this.worker.postMessage({ id, command, page, bbox }); });
|
|
89
91
|
return command === 'page' ? nativePageSchema.parse(value) : renderSchema.parse(value);
|
|
90
92
|
}
|
|
91
93
|
close() {
|
|
@@ -114,9 +116,13 @@ export function pageRange(start, end, count) {
|
|
|
114
116
|
export async function inspectPdf(file, config, signal) {
|
|
115
117
|
const session = new PdfSession(file, config, signal);
|
|
116
118
|
try {
|
|
117
|
-
const info = await session.ready
|
|
119
|
+
const info = await session.ready;
|
|
120
|
+
const n = info.pageCount;
|
|
121
|
+
const samples = [...new Set([1, Math.min(3, n), Math.ceil(n / 3), Math.ceil(n * 2 / 3), n])];
|
|
122
|
+
const pages = [];
|
|
118
123
|
for (const page of samples) {
|
|
119
|
-
const value = await session.call('page', page)
|
|
124
|
+
const value = await session.call('page', page);
|
|
125
|
+
const text = value.spans.filter(s => !s.artifact).map(s => s.text).join('');
|
|
120
126
|
pages.push({ page, chars: Array.from(text).length, hasGraphics: value.drawings.length > 0, possibleOcr: text.length < 30 && value.drawings.length > 0, hasStructure: Boolean(value.tree) });
|
|
121
127
|
}
|
|
122
128
|
return { sourceFormat: 'pdf', pageCount: n, outlineCount: info.outline.length, samples: pages, possibleOcr: pages.some(p => p.possibleOcr), note: '仅抽样预检;正式构建逐页核对,图表及扫描识别共用模型预算。' };
|
|
@@ -16,7 +16,9 @@ export async function restoreLayout(page, layout, file, job, models, signal) {
|
|
|
16
16
|
throw new Error('此页需要版面识别/OCR,请配置图片模型并启用图片理解');
|
|
17
17
|
if (!job.visionCapability)
|
|
18
18
|
await models.preflight(job, signal);
|
|
19
|
-
const spans = page.spans.filter(s => !s.artifact)
|
|
19
|
+
const spans = page.spans.filter(s => !s.artifact);
|
|
20
|
+
const ocr = layout.ocr;
|
|
21
|
+
const badIds = new Set(spans.filter(s => (s.text.includes('\ufffd') || s.text.includes('\0'))).map(s => s.id));
|
|
20
22
|
if (chars(JSON.stringify(spans)) > 90000)
|
|
21
23
|
throw new Error('此页原生文字对象超过版面识别输入上限,请缩小页范围或使用专用解析器');
|
|
22
24
|
const prepared = await prepareImage(models.ctx, file, job.config, signal);
|
|
@@ -29,7 +31,8 @@ export async function restoreLayout(page, layout, file, job, models, signal) {
|
|
|
29
31
|
const value = assignmentSchema.parse(raw);
|
|
30
32
|
if (!Array.isArray(value.blocks) || !value.blocks.length || value.blocks.length > 2000 || !Array.isArray(value.uncertainties) || value.uncertainties.length > 100)
|
|
31
33
|
throw new Error('PDF 版面识别 JSON 无效');
|
|
32
|
-
const used = new Set()
|
|
34
|
+
const used = new Set();
|
|
35
|
+
const byId = new Map(spans.map(s => [s.id, s]));
|
|
33
36
|
const take = (row, allowEmpty = false) => {
|
|
34
37
|
if (ocr && (!row.spanIds || Array.isArray(row.spanIds) && !row.spanIds.length)) {
|
|
35
38
|
if (typeof row.text !== 'string' || chars(row.text) > 30000 || !allowEmpty && !row.text.trim())
|
|
@@ -97,7 +100,9 @@ export async function restoreLayout(page, layout, file, job, models, signal) {
|
|
|
97
100
|
}
|
|
98
101
|
// The assigned strings must lie inside their claimed region. No numeric text rewriting is accepted.
|
|
99
102
|
for (const id of block.spanIds || []) {
|
|
100
|
-
const s = required(byId.get(id))
|
|
103
|
+
const s = required(byId.get(id));
|
|
104
|
+
const x = (s.bbox[0] + s.bbox[2]) / 2;
|
|
105
|
+
const y = (s.bbox[1] + s.bbox[3]) / 2;
|
|
101
106
|
if (x < b[0] - 2 || x > b[2] + 2 || y < b[1] - 2 || y > b[3] + 2)
|
|
102
107
|
throw new Error('PDF 文字与识别区域不匹配');
|
|
103
108
|
}
|