@dsh-libing/knowledge-base 0.1.0 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/CHANGELOG.md +20 -0
  2. package/README.md +12 -2
  3. package/lib/agent-access.d.ts +23 -0
  4. package/lib/agent-access.js +28 -0
  5. package/lib/chunks.js +27 -5
  6. package/lib/client-types/client/JobDetails.d.ts +16 -0
  7. package/lib/client-types/client/JobsPanel.d.ts +6 -1
  8. package/lib/client-types/client/QualityReport.d.ts +3 -2
  9. package/lib/client-types/client/api.d.ts +83 -52
  10. package/lib/client-types/client/locales.d.ts +2 -0
  11. package/lib/client-types/client/useKnowledge.d.ts +30 -15
  12. package/lib/client-types/contract.d.ts +83 -52
  13. package/lib/client-types/data.d.ts +11 -0
  14. package/lib/client-types/settings.d.ts +4 -0
  15. package/lib/client-types/shared.d.ts +17 -0
  16. package/lib/client.js +40 -40
  17. package/lib/config.js +9 -1
  18. package/lib/contract.d.ts +33 -2
  19. package/lib/contract.js +2 -1
  20. package/lib/data.d.ts +11 -0
  21. package/lib/data.js +2 -0
  22. package/lib/document/convert.js +52 -16
  23. package/lib/document/pdf-layout.js +45 -10
  24. package/lib/document/pdf-session.js +10 -4
  25. package/lib/document/pdf-vision.js +8 -3
  26. package/lib/document/pdf-worker.js +76 -21
  27. package/lib/document/pdf.js +39 -9
  28. package/lib/document/zip.js +14 -6
  29. package/lib/figures/image-client.d.ts +5 -0
  30. package/lib/figures/image-client.js +63 -0
  31. package/lib/figures/image-gate.d.ts +12 -0
  32. package/lib/figures/image-gate.js +39 -0
  33. package/lib/figures/image-protocol.d.ts +19 -0
  34. package/lib/figures/image-protocol.js +9 -0
  35. package/lib/figures/image-worker.d.ts +1 -0
  36. package/lib/figures/image-worker.js +69 -0
  37. package/lib/figures/process.d.ts +1 -1
  38. package/lib/figures/process.js +19 -38
  39. package/lib/index.js +6 -2
  40. package/lib/indexes.js +17 -6
  41. package/lib/job-storage.d.ts +264 -0
  42. package/lib/job-storage.js +118 -0
  43. package/lib/jobs.d.ts +782 -1489
  44. package/lib/jobs.js +64 -31
  45. package/lib/models.d.ts +1 -1
  46. package/lib/models.js +15 -9
  47. package/lib/retrieval.js +44 -13
  48. package/lib/revision-cache.d.ts +24 -0
  49. package/lib/revision-cache.js +100 -0
  50. package/lib/routes.js +66 -20
  51. package/lib/runtime.d.ts +4 -0
  52. package/lib/runtime.js +15 -4
  53. package/lib/settings.d.ts +4 -0
  54. package/lib/settings.js +5 -2
  55. package/lib/shared.d.ts +17 -0
  56. package/lib/shared.js +3 -0
  57. package/lib/skills.js +6 -3
  58. package/lib/store.d.ts +101 -5
  59. package/lib/store.js +78 -29
  60. package/lib/tools.js +12 -4
  61. package/lib/util.js +2 -1
  62. package/package.json +1 -1
package/lib/config.js CHANGED
@@ -10,4 +10,12 @@ export function resolveConfig(raw = {}) {
10
10
  return settingsSchema.parse({ ...raw, dataRoot });
11
11
  }
12
12
  const defaults = resolveConfig();
13
- export const Config = Schema.object(Object.fromEntries(Object.entries(defaults).filter(([key]) => !['skipImages', 'acceptPartial'].includes(key)).map(([key, value]) => [key, typeof value === 'number' ? Schema.number().min(1).default(value) : typeof value === 'string' ? Schema.string().default(value) : Schema.boolean().default(value)])));
13
+ export const Config = Schema.object(Object.fromEntries(Object.entries(defaults).filter(([key]) => !['skipImages', 'acceptPartial'].includes(key)).map(([key, value]) => [key, (() => {
14
+ if (typeof value === 'number') {
15
+ return Schema.number().min(1).default(value);
16
+ }
17
+ if (typeof value === 'string') {
18
+ return Schema.string().default(value);
19
+ }
20
+ return Schema.boolean().default(value);
21
+ })()])));
package/lib/contract.d.ts CHANGED
@@ -119,8 +119,12 @@ export declare const publicJobSchema: z.ZodObject<{
119
119
  modelTimeoutMs: z.ZodDefault<z.ZodNumber>;
120
120
  modelAttempts: z.ZodDefault<z.ZodNumber>;
121
121
  visionConcurrency: z.ZodDefault<z.ZodNumber>;
122
+ maxModelCallsLimit: z.ZodDefault<z.ZodNumber>;
123
+ maxModelTokensLimit: z.ZodDefault<z.ZodNumber>;
124
+ uploadRetentionDays: z.ZodDefault<z.ZodNumber>;
122
125
  maxModelCalls: z.ZodDefault<z.ZodNumber>;
123
126
  maxModelTokens: z.ZodDefault<z.ZodNumber>;
127
+ imageTimeoutMs: z.ZodDefault<z.ZodNumber>;
124
128
  visionMaxEdge: z.ZodDefault<z.ZodNumber>;
125
129
  visionTileSize: z.ZodDefault<z.ZodNumber>;
126
130
  visionMaxTiles: z.ZodDefault<z.ZodNumber>;
@@ -189,6 +193,13 @@ export declare const publicJobSchema: z.ZodObject<{
189
193
  chunkId: z.ZodString;
190
194
  }, z.core.$strip>>;
191
195
  }, z.core.$strip>;
196
+ pendingCalls: z.ZodDefault<z.ZodArray<z.ZodNumber>>;
197
+ legacyUsageCount: z.ZodDefault<z.ZodNumber>;
198
+ legacyCallCount: z.ZodDefault<z.ZodNumber>;
199
+ reportSummary: z.ZodOptional<z.ZodObject<{
200
+ issues: z.ZodNumber;
201
+ blocking: z.ZodNumber;
202
+ }, z.core.$strip>>;
192
203
  failures: z.ZodArray<z.ZodObject<{
193
204
  at: z.ZodString;
194
205
  stage: z.ZodEnum<{
@@ -437,8 +448,12 @@ export declare const stateSchema: z.ZodObject<{
437
448
  modelTimeoutMs: z.ZodDefault<z.ZodNumber>;
438
449
  modelAttempts: z.ZodDefault<z.ZodNumber>;
439
450
  visionConcurrency: z.ZodDefault<z.ZodNumber>;
451
+ maxModelCallsLimit: z.ZodDefault<z.ZodNumber>;
452
+ maxModelTokensLimit: z.ZodDefault<z.ZodNumber>;
453
+ uploadRetentionDays: z.ZodDefault<z.ZodNumber>;
440
454
  maxModelCalls: z.ZodDefault<z.ZodNumber>;
441
455
  maxModelTokens: z.ZodDefault<z.ZodNumber>;
456
+ imageTimeoutMs: z.ZodDefault<z.ZodNumber>;
442
457
  visionMaxEdge: z.ZodDefault<z.ZodNumber>;
443
458
  visionTileSize: z.ZodDefault<z.ZodNumber>;
444
459
  visionMaxTiles: z.ZodDefault<z.ZodNumber>;
@@ -507,6 +522,13 @@ export declare const stateSchema: z.ZodObject<{
507
522
  chunkId: z.ZodString;
508
523
  }, z.core.$strip>>;
509
524
  }, z.core.$strip>;
525
+ pendingCalls: z.ZodDefault<z.ZodArray<z.ZodNumber>>;
526
+ legacyUsageCount: z.ZodDefault<z.ZodNumber>;
527
+ legacyCallCount: z.ZodDefault<z.ZodNumber>;
528
+ reportSummary: z.ZodOptional<z.ZodObject<{
529
+ issues: z.ZodNumber;
530
+ blocking: z.ZodNumber;
531
+ }, z.core.$strip>>;
510
532
  failures: z.ZodArray<z.ZodObject<{
511
533
  at: z.ZodString;
512
534
  stage: z.ZodEnum<{
@@ -557,6 +579,9 @@ export declare const stateSchema: z.ZodObject<{
557
579
  outputPath: z.ZodNullable<z.ZodString>;
558
580
  reused: z.ZodOptional<z.ZodBoolean>;
559
581
  }, z.core.$strip>>;
582
+ jobTotal: z.ZodDefault<z.ZodNumber>;
583
+ jobOffset: z.ZodDefault<z.ZodNumber>;
584
+ activeJobs: z.ZodDefault<z.ZodNumber>;
560
585
  dataRoot: z.ZodString;
561
586
  config: z.ZodObject<{
562
587
  chunkTargetChars: z.ZodNumber;
@@ -669,8 +694,8 @@ export declare const treeDocumentSchema: z.ZodObject<{
669
694
  reason: z.ZodOptional<z.ZodString>;
670
695
  }, z.core.$strip>>>;
671
696
  chunks: z.ZodArray<z.ZodObject<{
672
- summary: z.ZodDefault<z.ZodString>;
673
697
  path: z.ZodString;
698
+ summary: z.ZodDefault<z.ZodString>;
674
699
  title: z.ZodString;
675
700
  headingPath: z.ZodArray<z.ZodString>;
676
701
  oversizedReason: z.ZodOptional<z.ZodString>;
@@ -972,8 +997,8 @@ export declare const treeSchema: z.ZodObject<{
972
997
  reason: z.ZodOptional<z.ZodString>;
973
998
  }, z.core.$strip>>>;
974
999
  chunks: z.ZodArray<z.ZodObject<{
975
- summary: z.ZodDefault<z.ZodString>;
976
1000
  path: z.ZodString;
1001
+ summary: z.ZodDefault<z.ZodString>;
977
1002
  title: z.ZodString;
978
1003
  headingPath: z.ZodArray<z.ZodString>;
979
1004
  oversizedReason: z.ZodOptional<z.ZodString>;
@@ -1195,3 +1220,9 @@ export declare const uploadResultSchema: z.ZodObject<{
1195
1220
  filename: z.ZodString;
1196
1221
  bytes: z.ZodNumber;
1197
1222
  }, z.core.$strip>;
1223
+ export declare const jobUsagePageSchema: z.ZodObject<{
1224
+ records: z.ZodArray<z.ZodJSONSchema>;
1225
+ offset: z.ZodNumber;
1226
+ total: z.ZodNumber;
1227
+ next: z.ZodNullable<z.ZodNumber>;
1228
+ }, z.core.$strip>;
package/lib/contract.js CHANGED
@@ -4,7 +4,7 @@ import { jobSchema, registrySchema, librarySchema, publishedDocumentSchema, repo
4
4
  export const publicJobSchema = jobSchema.omit({ schema: true, overlays: true }).extend({ outputPath: z.string().nullable(), reused: z.boolean().optional() });
5
5
  export const locationSchema = z.object({ kbPath: z.string(), revision: z.string().nullable(), wikiPath: z.string().nullable(), indexPath: z.string().nullable() });
6
6
  export const libraryViewSchema = librarySchema.extend({ location: locationSchema, summary: z.object({ revision: z.string().nullable(), documentCount: z.number(), chunkCount: z.number(), coverageIncomplete: z.boolean() }) });
7
- export const stateSchema = z.object({ hostVersion: z.string(), kbs: z.array(libraryViewSchema), deletedKbs: z.array(librarySchema), activities: registrySchema.shape.activities, jobs: z.array(publicJobSchema), dataRoot: z.string(), config: z.object({ chunkTargetChars: z.number(), visionProvider: z.string(), visionModel: z.string(), maxInputBytes: z.number() }) });
7
+ export const stateSchema = z.object({ hostVersion: z.string(), kbs: z.array(libraryViewSchema), deletedKbs: z.array(librarySchema), activities: registrySchema.shape.activities, jobs: z.array(publicJobSchema), jobTotal: z.number().int().nonnegative().default(0), jobOffset: z.number().int().nonnegative().default(0), activeJobs: z.number().int().nonnegative().default(0), dataRoot: z.string(), config: z.object({ chunkTargetChars: z.number(), visionProvider: z.string(), visionModel: z.string(), maxInputBytes: z.number() }) });
8
8
  export const treeDocumentSchema = publishedDocumentSchema.extend({ chunks: z.array(publishedDocumentSchema.shape.chunks.element.omit({ segments: true, anchors: true })), figures: z.array(publishedDocumentSchema.shape.figures.element.extend({ expectedHash: z.string().optional() })) });
9
9
  export const treeSchema = z.object({ kbId: z.string(), revision: z.string(), location: locationSchema.optional(), previewJobId: z.string().optional(), coverageIncomplete: z.boolean(), report: reportSchema, documents: z.array(treeDocumentSchema) });
10
10
  export const readSchema = z.object({ kbId: z.string(), chunkId: z.string(), revision: z.string(), text: z.string(), complete: z.boolean(), truncated: z.boolean().optional(), previewJobId: z.string().optional(),
@@ -16,3 +16,4 @@ export const modelsSchema = z.object({ providers: z.array(z.object({ id: z.strin
16
16
  export const schemaResultSchema = z.object({ content: z.string(), hash: z.string() });
17
17
  export const inspectSchema = z.object({ sourceFormat: z.literal('pdf'), pageCount: z.number(), outlineCount: z.number(), samples: z.array(z.object({ page: z.number(), chars: z.number(), hasGraphics: z.boolean(), possibleOcr: z.boolean(), hasStructure: z.boolean() })), possibleOcr: z.boolean(), note: z.string() });
18
18
  export const uploadResultSchema = z.object({ receiptId: z.string(), filename: z.string(), bytes: z.number() });
19
+ export const jobUsagePageSchema = z.object({ records: z.array(z.json()), offset: z.number().int().nonnegative(), total: z.number().int().nonnegative(), next: z.number().int().nonnegative().nullable() });
package/lib/data.d.ts CHANGED
@@ -1528,8 +1528,12 @@ export declare const jobSchema: z.ZodObject<{
1528
1528
  modelTimeoutMs: z.ZodDefault<z.ZodNumber>;
1529
1529
  modelAttempts: z.ZodDefault<z.ZodNumber>;
1530
1530
  visionConcurrency: z.ZodDefault<z.ZodNumber>;
1531
+ maxModelCallsLimit: z.ZodDefault<z.ZodNumber>;
1532
+ maxModelTokensLimit: z.ZodDefault<z.ZodNumber>;
1533
+ uploadRetentionDays: z.ZodDefault<z.ZodNumber>;
1531
1534
  maxModelCalls: z.ZodDefault<z.ZodNumber>;
1532
1535
  maxModelTokens: z.ZodDefault<z.ZodNumber>;
1536
+ imageTimeoutMs: z.ZodDefault<z.ZodNumber>;
1533
1537
  visionMaxEdge: z.ZodDefault<z.ZodNumber>;
1534
1538
  visionTileSize: z.ZodDefault<z.ZodNumber>;
1535
1539
  visionMaxTiles: z.ZodDefault<z.ZodNumber>;
@@ -1599,6 +1603,13 @@ export declare const jobSchema: z.ZodObject<{
1599
1603
  chunkId: z.ZodString;
1600
1604
  }, z.core.$strip>>;
1601
1605
  }, z.core.$strip>;
1606
+ pendingCalls: z.ZodDefault<z.ZodArray<z.ZodNumber>>;
1607
+ legacyUsageCount: z.ZodDefault<z.ZodNumber>;
1608
+ legacyCallCount: z.ZodDefault<z.ZodNumber>;
1609
+ reportSummary: z.ZodOptional<z.ZodObject<{
1610
+ issues: z.ZodNumber;
1611
+ blocking: z.ZodNumber;
1612
+ }, z.core.$strip>>;
1602
1613
  usage: z.ZodObject<{
1603
1614
  calls: z.ZodNumber;
1604
1615
  tokens: z.ZodNumber;
package/lib/data.js CHANGED
@@ -69,6 +69,8 @@ export const jobSchema = z.object({ jobId: idSchema, kbId: idSchema, docId: idSc
69
69
  progress: z.object({ totalPages: z.number().optional(), parsedPages: z.number().optional(), ocrPages: z.number().optional(), tables: z.number().optional(), blocks: z.number().optional(), currentPage: z.number().optional(),
70
70
  totalFigures: z.number().optional(), completedFigures: z.number().optional(), inflightFigures: z.array(z.object({ occurrenceId: z.string(), caption: z.string() })).optional(),
71
71
  totalChunks: z.number().optional(), indexedChunks: z.number().optional(), currentChunk: z.object({ title: z.string(), headingPath: z.array(z.string()), chunkId: z.string() }).optional() }),
72
+ pendingCalls: z.array(z.number().int().positive()).max(16).default([]), legacyUsageCount: z.number().int().nonnegative().default(0), legacyCallCount: z.number().int().nonnegative().default(0),
73
+ reportSummary: z.object({ issues: z.number().int().nonnegative(), blocking: z.number().int().nonnegative() }).optional(),
72
74
  usage: usageSchema, failures: z.array(z.object({ at: z.string(), stage: jobStateSchema, error: z.string() })), events: z.array(z.object({ at: z.string(), state: jobStateSchema })).optional(),
73
75
  figureRefresh: z.object({ occurrenceId: z.string(), token: z.string() }).nullable().optional(), visionCapability: z.string().optional(), parserVersion: z.string().optional(),
74
76
  pageStart: z.number().optional(), pageEnd: z.number().optional(), pdf: z.object({ pageCount: z.number(), selectedPages: rangeSchema, outlineCount: z.number().optional() }).optional(),
@@ -12,7 +12,15 @@ import { mkdtemp, readFile, rm } from 'node:fs/promises';
12
12
  import { tmpdir } from 'node:os';
13
13
  const parser = new XMLParser({ ignoreAttributes: false, removeNSPrefix: true, parseTagValue: false, trimValues: false });
14
14
  const xmlSchema = z.lazy(() => z.union([z.string(), z.number(), z.boolean(), z.null(), z.array(xmlSchema), z.record(z.string(), xmlSchema)]));
15
- const list = (value) => value === undefined ? [] : Array.isArray(value) ? value : [value];
15
+ const list = (value) => {
16
+ if (value === undefined) {
17
+ return [];
18
+ }
19
+ if (Array.isArray(value)) {
20
+ return value;
21
+ }
22
+ return [value];
23
+ };
16
24
  const object = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {};
17
25
  function child(value, ...keys) {
18
26
  for (const key of keys)
@@ -43,8 +51,13 @@ function wordText(value) {
43
51
  }
44
52
  const normalize = (value) => value.replace(/\s/gu, '');
45
53
  function audit(files) {
46
- const doc = parseXml(required(files['word/document.xml'])), styles = files['word/styles.xml'] ? parseXml(files['word/styles.xml']) : {};
47
- const warnings = [], styleLevels = {}, headingLevels = {}, outlineLevels = {}, anchors = new Set();
54
+ const doc = parseXml(required(files['word/document.xml']));
55
+ const styles = files['word/styles.xml'] ? parseXml(files['word/styles.xml']) : {};
56
+ const warnings = [];
57
+ const styleLevels = {};
58
+ const headingLevels = {};
59
+ const outlineLevels = {};
60
+ const anchors = new Set();
48
61
  const styleRows = list(child(styles, 'styles', 'style'));
49
62
  const resolveLevel = (styleId, seen = new Set()) => {
50
63
  if (seen.has(styleId) || !styleId)
@@ -53,19 +66,33 @@ function audit(files) {
53
66
  const row = styleRows.find(v => xmlString(child(v, '@_styleId')) === styleId);
54
67
  if (!row)
55
68
  return;
56
- const explicit = child(row, 'pPr', 'outlineLvl', '@_val'), name = xmlString(child(row, 'name', '@_val')) || styleId, common = /^(?:heading|标题)\s*([1-9])$/iu.exec(name);
57
- return explicit !== undefined && Number(explicit) < 9 ? Number(explicit) + 1 : common ? Number(common[1]) : resolveLevel(xmlString(child(row, 'basedOn', '@_val')), seen);
69
+ const explicit = child(row, 'pPr', 'outlineLvl', '@_val');
70
+ const name = xmlString(child(row, 'name', '@_val')) || styleId;
71
+ const common = /^(?:heading|标题)\s*([1-9])$/iu.exec(name);
72
+ if (explicit !== undefined && Number(explicit) < 9) {
73
+ return Number(explicit) + 1;
74
+ }
75
+ if (common) {
76
+ return Number(common[1]);
77
+ }
78
+ return resolveLevel(xmlString(child(row, 'basedOn', '@_val')), seen);
58
79
  };
59
80
  for (const row of styleRows) {
60
- const styleId = xmlString(child(row, '@_styleId')), level = resolveLevel(styleId);
81
+ const styleId = xmlString(child(row, '@_styleId'));
82
+ const level = resolveLevel(styleId);
61
83
  if (level)
62
84
  styleLevels[styleId] = level;
63
85
  }
64
- let expectedFigures = 0, expectedTables = 0, textBoxes = 0, footnoteRefs = 0;
86
+ let expectedFigures = 0;
87
+ let expectedTables = 0;
88
+ let textBoxes = 0;
89
+ let footnoteRefs = 0;
65
90
  const expectedHeadings = [];
66
91
  visit(doc, (key, value) => {
67
92
  if (key === 'p') {
68
- const outline = child(value, 'pPr', 'outlineLvl', '@_val'), style = xmlString(child(value, 'pPr', 'pStyle', '@_val')), level = outline !== undefined && Number(outline) < 9 ? Number(outline) + 1 : styleLevels[style];
93
+ const outline = child(value, 'pPr', 'outlineLvl', '@_val');
94
+ const style = xmlString(child(value, 'pPr', 'pStyle', '@_val'));
95
+ const level = outline !== undefined && Number(outline) < 9 ? Number(outline) + 1 : styleLevels[style];
69
96
  if (outline !== undefined && Number(outline) < 9)
70
97
  outlineLevels[normalize(wordText(value))] = Number(outline) + 1;
71
98
  if (level && !/^TOC/iu.test(style)) {
@@ -119,7 +146,8 @@ function safeLink(value) { return /^(?:https?:|mailto:|#)/i.test(value || '') ?
119
146
  function inline(node, internalRefs) {
120
147
  if ('value' in node)
121
148
  return node.value.replace(/([\\`*[\]<>])/g, '\\$1');
122
- const t = tag(node), body = children(node).map(n => inline(n, internalRefs)).join('');
149
+ const t = tag(node);
150
+ const body = children(node).map(n => inline(n, internalRefs)).join('');
123
151
  if (['script', 'style', 'iframe', 'object'].includes(t))
124
152
  return '';
125
153
  if (t === 'img')
@@ -159,8 +187,10 @@ function tableHtml(rows) {
159
187
  }
160
188
  export async function convertDocx(buffer, destination, config, signal) {
161
189
  checkAbort(signal);
162
- const files = unpackDocx(buffer, config), originalAudit = audit(files);
163
- const assets = [], warnings = [...originalAudit.warnings];
190
+ const files = unpackDocx(buffer, config);
191
+ const originalAudit = audit(files);
192
+ const assets = [];
193
+ const warnings = [...originalAudit.warnings];
164
194
  const mimeExtension = { 'image/png': 'png', 'image/jpeg': 'jpg', 'image/gif': 'gif', 'image/webp': 'webp', 'image/bmp': 'bmp', 'image/tiff': 'tif', 'image/x-emf': 'emf', 'image/x-wmf': 'wmf', 'image/svg+xml': 'svg' };
165
195
  const addAsset = async (data, mime, origin) => {
166
196
  const assetId = `a-${hash(data).slice(0, 20)}`;
@@ -174,7 +204,8 @@ export async function convertDocx(buffer, destination, config, signal) {
174
204
  };
175
205
  const rawMedia = Object.entries(files).filter(([name]) => name.startsWith('word/media/'));
176
206
  for (const [name, bytes] of rawMedia) {
177
- const ext = (name.split('.').at(-1) || '').toLowerCase(), mime = Object.entries(mimeExtension).find(([, x]) => x === ext || ext === 'jpeg' && x === 'jpg')?.[0] || 'application/octet-stream';
207
+ const ext = (name.split('.').at(-1) || '').toLowerCase();
208
+ const mime = Object.entries(mimeExtension).find(([, x]) => x === ext || ext === 'jpeg' && x === 'jpg')?.[0] || 'application/octet-stream';
178
209
  await addAsset(Buffer.from(bytes), mime, name);
179
210
  }
180
211
  const result = await mammoth.convertToHtml({ buffer }, {
@@ -208,11 +239,14 @@ export async function convertDocx(buffer, destination, config, signal) {
208
239
  // Unmapped styles lose formatting, not paragraph text; heading coverage is audited separately.
209
240
  blocking: !/^Unrecognised (?:paragraph|run) style:/i.test(m.message) && /image|unrecognised|unsupported/i.test(m.message),
210
241
  })));
211
- const fragment = parseFragment(result.value), blocks = [], internalRefs = [];
242
+ const fragment = parseFragment(result.value);
243
+ const blocks = [];
244
+ const internalRefs = [];
212
245
  let figureCount = 0;
213
246
  const add = (value) => { blocks.push(makeBlock({ blockId: `b-${String(blocks.length + 1).padStart(6, '0')}`, ...value })); return required(blocks.at(-1)); };
214
247
  const addInline = (node, type = 'paragraph', prefix = '') => {
215
- const body = inline(node, internalRefs), pieces = body.split(String.fromCharCode(1)).filter(Boolean);
248
+ const body = inline(node, internalRefs);
249
+ const pieces = body.split(String.fromCharCode(1)).filter(Boolean);
216
250
  for (const piece of pieces) {
217
251
  if (piece.startsWith('asset:')) {
218
252
  figureCount++;
@@ -227,7 +261,8 @@ export async function convertDocx(buffer, destination, config, signal) {
227
261
  if (/^h[1-9]$/.test(t))
228
262
  add({ type: 'heading', level: Number(t[1]), text: htmlText(node).trim(), anchor: attr(node, 'id') || null });
229
263
  else if (t === 'table') {
230
- const rows = tableCells(node), table = add({ type: 'table', rows, text: rows.map(row => row.map(c => c.text).join(' | ')).join('\n'), markdown: tableHtml(rows) });
264
+ const rows = tableCells(node);
265
+ const table = add({ type: 'table', rows, text: rows.map(row => row.map(c => c.text).join(' | ')).join('\n'), markdown: tableHtml(rows) });
231
266
  const scan = (n) => {
232
267
  if (tag(n) === 'img') {
233
268
  const assetId = (attr(n, 'src') || '').replace(/^asset:/, '');
@@ -241,7 +276,8 @@ export async function convertDocx(buffer, destination, config, signal) {
241
276
  scan(node);
242
277
  }
243
278
  else if (t === 'p') {
244
- const t = htmlText(node).trim(), previous = blocks.at(-1);
279
+ const t = htmlText(node).trim();
280
+ const previous = blocks.at(-1);
245
281
  if (/^(?:图|Figure)\s*[A-Za-z]?\d[\d.\-–]*/i.test(t) && previous?.type === 'figure') {
246
282
  previous.caption = t;
247
283
  add({ type: 'caption', text: t, figureRef: previous.occurrenceId });
@@ -19,7 +19,8 @@ export function lines(spans) {
19
19
  }
20
20
  return rows.sort((a, b) => a.y - b.y).map(r => {
21
21
  const items = r.spans.sort((a, b) => readingBox(a)[0] - readingBox(b)[0]);
22
- let text = '', last;
22
+ let text = '';
23
+ let last;
23
24
  for (const s of items) {
24
25
  if (last && readingBox(s)[0] - readingBox(last)[2] > Math.max(1, s.fontSize * .18) && !/\s$/.test(text))
25
26
  text += ' ';
@@ -83,16 +84,23 @@ function geometricTables(page, spans, occupied) {
83
84
  group.push(required(rows[++i]));
84
85
  if (group.length < 2)
85
86
  continue;
86
- const ids = new Set(), cells = group.map(r => r.cells.map(b => { const ss = spans.filter(s => { const x = (s.bbox[0] + s.bbox[2]) / 2, y = (s.bbox[1] + s.bbox[3]) / 2; return x > b[0] && x < b[2] && y > b[1] && y < b[3]; }); ss.forEach(s => ids.add(s.id)); return { text: textOf(ss), spanIds: ss.map(s => s.id), bbox: b, colspan: 1, rowspan: 1 }; }));
87
+ const ids = new Set();
88
+ const cells = group.map(r => r.cells.map(b => { const ss = spans.filter(s => { const x = (s.bbox[0] + s.bbox[2]) / 2; const y = (s.bbox[1] + s.bbox[3]) / 2; return x > b[0] && x < b[2] && y > b[1] && y < b[3]; }); ss.forEach(s => ids.add(s.id)); return { text: textOf(ss), spanIds: ss.map(s => s.id), bbox: b, colspan: 1, rowspan: 1 }; }));
87
89
  if (cells.some(r => r.every(c => !c.text)))
88
90
  continue;
89
- const bbox = required(union(group.flatMap(r => r.cells))), identified = /field|offset|bits|name|description|字段|偏移|名称|参数|描述|位范围/i.test(required(cells[0]).map(c => c.text).join(' '));
91
+ const bbox = required(union(group.flatMap(r => r.cells)));
92
+ const identified = /field|offset|bits|name|description|字段|偏移|名称|参数|描述|位范围/i.test(required(cells[0]).map(c => c.text).join(' '));
90
93
  tables.push({ rows: cells, bbox, ids, identified });
91
94
  }
92
95
  return tables;
93
96
  }
94
97
  export function nativeLayout(page, info) {
95
- const byMarked = new Map(), used = new Set(), blocks = [], figures = [], warnings = (page.warnings || []).map(message => ({ code: 'pdf-engine-warning', page: page.page, message: `PDF 第 ${page.page} 页解析器警告:${message}`, blocking: true })), excluded = page.spans.filter(s => s.artifact).map(s => ({ kind: 'tagged-artifact', text: s.text, bbox: s.bbox }));
98
+ const byMarked = new Map();
99
+ const used = new Set();
100
+ const blocks = [];
101
+ const figures = [];
102
+ const warnings = (page.warnings || []).map(message => ({ code: 'pdf-engine-warning', page: page.page, message: `PDF 第 ${page.page} 页解析器警告:${message}`, blocking: true }));
103
+ const excluded = page.spans.filter(s => s.artifact).map(s => ({ kind: 'tagged-artifact', text: s.text, bbox: s.bbox }));
96
104
  const textSpans = page.spans.filter(s => !s.artifact);
97
105
  for (const s of textSpans) {
98
106
  if (!byMarked.has(s.mcid))
@@ -128,7 +136,8 @@ export function nativeLayout(page, info) {
128
136
  if (!rows.length)
129
137
  return;
130
138
  const irregular = rows.some(r => r.length !== required(rows[0]).length);
131
- const textBox = required(union(spans.map(s => s.bbox))), nearby = page.drawings.filter(d => d.kind === 'path' && d.bbox[1] >= textBox[1] - 20 && d.bbox[3] <= textBox[3] + 20 && intersects(d.bbox, boundedBox(textBox, page, 10))).map(d => d.bbox);
139
+ const textBox = required(union(spans.map(s => s.bbox)));
140
+ const nearby = page.drawings.filter(d => d.kind === 'path' && d.bbox[1] >= textBox[1] - 20 && d.bbox[3] <= textBox[3] + 20 && intersects(d.bbox, boundedBox(textBox, page, 10))).map(d => d.bbox);
132
141
  const tableBox = boundedBox(required(union([textBox, ...nearby])), page, 3);
133
142
  add('table', spans, { source: source(tableBox), rows, markdown: tableMarkdown(rows), needsReview: irregular, caption: '', layoutBasis: 'structure-tags', ...textBox[3] > page.height * .85 && !closedTableBottom(page, textBox, tableBox) ? { openEnd: true } : {} });
134
143
  if (irregular)
@@ -136,7 +145,10 @@ export function nativeLayout(page, info) {
136
145
  return;
137
146
  }
138
147
  if (node.role === 'Figure') {
139
- const marked = ids(node), spans = get(node), boxes = [...page.drawings.filter(d => d.mcid !== null && marked.includes(d.mcid)).map(d => d.bbox), ...spans.map(s => s.bbox)], bbox = union(boxes);
148
+ const marked = ids(node);
149
+ const spans = get(node);
150
+ const boxes = [...page.drawings.filter(d => d.mcid !== null && marked.includes(d.mcid)).map(d => d.bbox), ...spans.map(s => s.bbox)];
151
+ const bbox = union(boxes);
140
152
  if (!bbox)
141
153
  return;
142
154
  consume(spans);
@@ -148,7 +160,15 @@ export function nativeLayout(page, info) {
148
160
  return;
149
161
  }
150
162
  if (['Code', 'Formula', 'Note'].includes(node.role || '')) {
151
- add(node.role === 'Code' ? 'code' : node.role === 'Formula' ? 'formula' : 'note', get(node));
163
+ add((() => {
164
+ if (node.role === 'Code') {
165
+ return 'code';
166
+ }
167
+ if (node.role === 'Formula') {
168
+ return 'formula';
169
+ }
170
+ return 'note';
171
+ })(), get(node));
152
172
  return;
153
173
  }
154
174
  if (node.role === 'P') {
@@ -194,8 +214,21 @@ export function nativeLayout(page, info) {
194
214
  const match = info.outline.find(o => {
195
215
  if (o.page !== page.page || normalize(o.title) !== normalize(b.text))
196
216
  return false;
197
- const candidates = blocks.filter(c => normalize(c.text) === normalize(o.title)), t = page.viewportTransform, axis = page.rotation % 180 ? 0 : 1;
198
- const target = o.y !== null && Number.isFinite(o.y) && t ? (axis === 0 ? required(t[2]) * o.y + required(t[4]) : required(t[3]) * o.y + required(t[5])) : null;
217
+ const candidates = blocks.filter(c => normalize(c.text) === normalize(o.title));
218
+ const t = page.viewportTransform;
219
+ const axis = page.rotation % 180 ? 0 : 1;
220
+ let target;
221
+ if (o.y !== null && Number.isFinite(o.y) && t) {
222
+ if (axis === 0) {
223
+ target = required(t[2]) * o.y + required(t[4]);
224
+ }
225
+ else {
226
+ target = required(t[3]) * o.y + required(t[5]);
227
+ }
228
+ }
229
+ else {
230
+ target = null;
231
+ }
199
232
  const score = (c) => target === null ? (c.type === 'heading' ? 0 : 100) + c.source.bbox[1] / 1000 : Math.abs(c.source.bbox[axis] - target);
200
233
  return candidates.sort((a, b) => score(a) - score(b))[0] === b && (target === null || score(b) < 100);
201
234
  });
@@ -225,7 +258,9 @@ export function nativeLayout(page, info) {
225
258
  blocks.sort((a, b) => a.source.bbox[1] - b.source.bbox[1] || a.source.bbox[0] - b.source.bbox[0]);
226
259
  const coverage = [...[...figures, ...blocks.filter(b => b.type === 'table')].map(b => boundedBox(b.source.bbox, page, 15)), ...excluded.map(e => boundedBox(e.bbox, page, 2))];
227
260
  const unexplained = page.drawings.filter(d => d.bbox[2] - d.bbox[0] > 25 && d.bbox[3] - d.bbox[1] > 12 && d.bbox[1] > 65 && d.bbox[3] < page.height - 40 && !coverage.some(b => intersects(b, d.bbox)));
228
- const raw = textSpans.map(s => s.text).join(''), garbled = textSpans.filter(s => (s.text.includes('\ufffd') || s.text.includes('\0'))), ocr = (raw.length < 30 && page.drawings.length > 0) || garbled.length > textSpans.length * .3;
261
+ const raw = textSpans.map(s => s.text).join('');
262
+ const garbled = textSpans.filter(s => (s.text.includes('\ufffd') || s.text.includes('\0')));
263
+ const ocr = (raw.length < 30 && page.drawings.length > 0) || garbled.length > textSpans.length * .3;
229
264
  if (garbled.length)
230
265
  warnings.push({ code: 'pdf-garbled-text', page: page.page, message: `PDF 第 ${page.page} 页存在缺字或乱码,需要核对识别区域`, blocking: true });
231
266
  const multiColumn = lines(textSpans.filter(s => !coverage.some(b => intersects(s.bbox, b)))).filter(l => l.spans.some((s, i) => i && s.bbox[0] - required(l.spans[i - 1]).bbox[2] > page.width * .13)).length >= 3;
@@ -24,7 +24,8 @@ export class PdfSession {
24
24
  this.signal = signal;
25
25
  checkAbort(signal);
26
26
  // Production uses the shipped JS worker. Source tests use the same TS through the installed tsx loader.
27
- const js = new URL('./pdf-worker.js', import.meta.url), source = !existsSync(fileURLToPath(js));
27
+ const js = new URL('./pdf-worker.js', import.meta.url);
28
+ const source = !existsSync(fileURLToPath(js));
28
29
  this.worker = new Worker(source ? new URL('./pdf-worker.ts', import.meta.url) : js, { workerData: { file, config: { maxInputBytes: config.maxInputBytes, pdfMaxPages: config.pdfMaxPages, pdfMaxPagePixels: config.pdfMaxPagePixels } }, resourceLimits: { maxOldGenerationSizeMb: 512 }, execArgv: source ? ['--import', 'tsx/esm'] : [] });
29
30
  const open = deferred();
30
31
  this.ready = open.promise;
@@ -85,7 +86,8 @@ export class PdfSession {
85
86
  throw this.error;
86
87
  if (this.closed)
87
88
  throw new Error('PDF 解析进程已关闭');
88
- const id = ++this.seq, value = await new Promise((resolve, reject) => { const timer = setTimeout(() => this.fail(new Error(`PDF 第 ${page} 页处理超时`)), this.config.pdfPageTimeoutMs); this.pending.set(id, { resolve, reject, timer }); this.worker.postMessage({ id, command, page, bbox }); });
89
+ const id = ++this.seq;
90
+ const value = await new Promise((resolve, reject) => { const timer = setTimeout(() => this.fail(new Error(`PDF 第 ${page} 页处理超时`)), this.config.pdfPageTimeoutMs); this.pending.set(id, { resolve, reject, timer }); this.worker.postMessage({ id, command, page, bbox }); });
89
91
  return command === 'page' ? nativePageSchema.parse(value) : renderSchema.parse(value);
90
92
  }
91
93
  close() {
@@ -114,9 +116,13 @@ export function pageRange(start, end, count) {
114
116
  export async function inspectPdf(file, config, signal) {
115
117
  const session = new PdfSession(file, config, signal);
116
118
  try {
117
- const info = await session.ready, n = info.pageCount, samples = [...new Set([1, Math.min(3, n), Math.ceil(n / 3), Math.ceil(n * 2 / 3), n])], pages = [];
119
+ const info = await session.ready;
120
+ const n = info.pageCount;
121
+ const samples = [...new Set([1, Math.min(3, n), Math.ceil(n / 3), Math.ceil(n * 2 / 3), n])];
122
+ const pages = [];
118
123
  for (const page of samples) {
119
- const value = await session.call('page', page), text = value.spans.filter(s => !s.artifact).map(s => s.text).join('');
124
+ const value = await session.call('page', page);
125
+ const text = value.spans.filter(s => !s.artifact).map(s => s.text).join('');
120
126
  pages.push({ page, chars: Array.from(text).length, hasGraphics: value.drawings.length > 0, possibleOcr: text.length < 30 && value.drawings.length > 0, hasStructure: Boolean(value.tree) });
121
127
  }
122
128
  return { sourceFormat: 'pdf', pageCount: n, outlineCount: info.outline.length, samples: pages, possibleOcr: pages.some(p => p.possibleOcr), note: '仅抽样预检;正式构建逐页核对,图表及扫描识别共用模型预算。' };
@@ -16,7 +16,9 @@ export async function restoreLayout(page, layout, file, job, models, signal) {
16
16
  throw new Error('此页需要版面识别/OCR,请配置图片模型并启用图片理解');
17
17
  if (!job.visionCapability)
18
18
  await models.preflight(job, signal);
19
- const spans = page.spans.filter(s => !s.artifact), ocr = layout.ocr, badIds = new Set(spans.filter(s => (s.text.includes('\ufffd') || s.text.includes('\0'))).map(s => s.id));
19
+ const spans = page.spans.filter(s => !s.artifact);
20
+ const ocr = layout.ocr;
21
+ const badIds = new Set(spans.filter(s => (s.text.includes('\ufffd') || s.text.includes('\0'))).map(s => s.id));
20
22
  if (chars(JSON.stringify(spans)) > 90000)
21
23
  throw new Error('此页原生文字对象超过版面识别输入上限,请缩小页范围或使用专用解析器');
22
24
  const prepared = await prepareImage(models.ctx, file, job.config, signal);
@@ -29,7 +31,8 @@ export async function restoreLayout(page, layout, file, job, models, signal) {
29
31
  const value = assignmentSchema.parse(raw);
30
32
  if (!Array.isArray(value.blocks) || !value.blocks.length || value.blocks.length > 2000 || !Array.isArray(value.uncertainties) || value.uncertainties.length > 100)
31
33
  throw new Error('PDF 版面识别 JSON 无效');
32
- const used = new Set(), byId = new Map(spans.map(s => [s.id, s]));
34
+ const used = new Set();
35
+ const byId = new Map(spans.map(s => [s.id, s]));
33
36
  const take = (row, allowEmpty = false) => {
34
37
  if (ocr && (!row.spanIds || Array.isArray(row.spanIds) && !row.spanIds.length)) {
35
38
  if (typeof row.text !== 'string' || chars(row.text) > 30000 || !allowEmpty && !row.text.trim())
@@ -97,7 +100,9 @@ export async function restoreLayout(page, layout, file, job, models, signal) {
97
100
  }
98
101
  // The assigned strings must lie inside their claimed region. No numeric text rewriting is accepted.
99
102
  for (const id of block.spanIds || []) {
100
- const s = required(byId.get(id)), x = (s.bbox[0] + s.bbox[2]) / 2, y = (s.bbox[1] + s.bbox[3]) / 2;
103
+ const s = required(byId.get(id));
104
+ const x = (s.bbox[0] + s.bbox[2]) / 2;
105
+ const y = (s.bbox[1] + s.bbox[3]) / 2;
101
106
  if (x < b[0] - 2 || x > b[2] + 2 || y < b[1] - 2 || y > b[3] + 2)
102
107
  throw new Error('PDF 文字与识别区域不匹配');
103
108
  }