@octopi-agent/engine 0.55.0 → 0.56.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/dist/config-schema/context.d.ts +38 -0
  2. package/dist/config-schema/context.js +39 -0
  3. package/dist/config-schema/engine.d.ts +37 -0
  4. package/dist/config-schema/engine.js +3 -1
  5. package/dist/config.d.ts +35 -0
  6. package/dist/harness/context/capabilities/document/backends/docx.d.ts +7 -0
  7. package/dist/harness/context/capabilities/document/backends/docx.js +67 -0
  8. package/dist/harness/context/capabilities/document/backends/office-parser.d.ts +7 -0
  9. package/dist/harness/context/capabilities/document/backends/office-parser.js +76 -0
  10. package/dist/harness/context/capabilities/document/backends/pdf.d.ts +7 -0
  11. package/dist/harness/context/capabilities/document/backends/pdf.js +86 -0
  12. package/dist/harness/context/capabilities/document/backends/plain.d.ts +7 -0
  13. package/dist/harness/context/capabilities/document/backends/plain.js +42 -0
  14. package/dist/harness/context/capabilities/document/backends/sheet.d.ts +7 -0
  15. package/dist/harness/context/capabilities/document/backends/sheet.js +90 -0
  16. package/dist/harness/context/capabilities/document/backends/xmind.d.ts +20 -0
  17. package/dist/harness/context/capabilities/document/backends/xmind.js +198 -0
  18. package/dist/harness/context/capabilities/document/errors.d.ts +27 -0
  19. package/dist/harness/context/capabilities/document/errors.js +33 -0
  20. package/dist/harness/context/capabilities/document/format.d.ts +38 -0
  21. package/dist/harness/context/capabilities/document/format.js +113 -0
  22. package/dist/harness/context/capabilities/document/html-to-md.d.ts +14 -0
  23. package/dist/harness/context/capabilities/document/html-to-md.js +97 -0
  24. package/dist/harness/context/capabilities/document/index.d.ts +21 -0
  25. package/dist/harness/context/capabilities/document/index.js +20 -0
  26. package/dist/harness/context/capabilities/document/legacy/factory.d.ts +13 -0
  27. package/dist/harness/context/capabilities/document/legacy/factory.js +23 -0
  28. package/dist/harness/context/capabilities/document/legacy/soffice.d.ts +32 -0
  29. package/dist/harness/context/capabilities/document/legacy/soffice.js +186 -0
  30. package/dist/harness/context/capabilities/document/port.d.ts +19 -0
  31. package/dist/harness/context/capabilities/document/port.js +400 -0
  32. package/dist/harness/context/capabilities/document/types.d.ts +149 -0
  33. package/dist/harness/context/capabilities/document/types.js +8 -0
  34. package/dist/harness/context/capabilities/index.d.ts +1 -0
  35. package/dist/harness/context/capabilities/index.js +1 -0
  36. package/dist/harness/extension/plugin-ecosystem/tools/builtin.d.ts +1 -0
  37. package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +10 -0
  38. package/dist/harness/extension/plugin-ecosystem/tools/document-tools.d.ts +64 -0
  39. package/dist/harness/extension/plugin-ecosystem/tools/document-tools.js +155 -0
  40. package/dist/harness/extension/plugin-ecosystem/tools/index.d.ts +1 -0
  41. package/dist/harness/extension/plugin-ecosystem/tools/index.js +1 -0
  42. package/dist/harness/extension/plugin-ecosystem/tools/tool-set.d.ts +2 -0
  43. package/dist/harness/extension/plugin-ecosystem/tools/tool-set.js +4 -1
  44. package/dist/harness/index.d.ts +2 -2
  45. package/dist/harness/index.js +1 -1
  46. package/dist/harness/knowledge/index-store.d.ts +7 -1
  47. package/dist/harness/knowledge/index-store.js +6 -0
  48. package/dist/harness/knowledge/ingest.d.ts +23 -0
  49. package/dist/harness/knowledge/ingest.js +81 -3
  50. package/dist/harness/session/attachments/extract.d.ts +3 -2
  51. package/dist/harness/session/attachments/extract.js +19 -6
  52. package/dist/harness/session/attachments/service.d.ts +5 -1
  53. package/dist/harness/session/attachments/service.js +44 -2
  54. package/dist/index.d.ts +2 -2
  55. package/dist/index.js +1 -1
  56. package/package.json +8 -3
@@ -14,6 +14,8 @@ import { hashContent, KnowledgeIndexStore } from './index-store.js';
14
14
  import { LocalFsFetcher, UrlFetcher } from './fetchers.js';
15
15
  import { ConnectorRegistry } from './connectors.js';
16
16
  import { ConnectorFetcher } from './connector-fetcher.js';
17
+ import { isDocumentPath } from '../context/capabilities/document/format.js';
18
+ import { isDocumentExtractError } from '../context/capabilities/document/errors.js';
17
19
  export class KnowledgeIngest extends EventEmitter {
18
20
  sources;
19
21
  index;
@@ -34,6 +36,7 @@ export class KnowledgeIngest extends EventEmitter {
34
36
  pollTickMs;
35
37
  pollMinIntervalMs;
36
38
  maxPollPerTick;
39
+ documentPort;
37
40
  lastEmbedAt = 0;
38
41
  pollTimer = null;
39
42
  running = 0;
@@ -61,8 +64,9 @@ export class KnowledgeIngest extends EventEmitter {
61
64
  this.embedConcurrency = options.embedConcurrency ?? options.parseConcurrency ?? 2;
62
65
  this.diskWatermarkAlert = options.diskWatermarkAlert ?? false;
63
66
  this.credentials = options.credentials ?? null;
67
+ this.documentPort = options.documentPort ?? null;
64
68
  this.localFetcher =
65
- options.fetchers?.local ?? new LocalFsFetcher((p) => this.adapters.shouldSkipPath(p));
69
+ options.fetchers?.local ?? new LocalFsFetcher((p) => this.shouldSkipFile(p));
66
70
  this.urlFetcher = options.fetchers?.url ?? new UrlFetcher();
67
71
  this.connectorFetcher =
68
72
  options.fetchers?.connector ??
@@ -74,6 +78,17 @@ export class KnowledgeIngest extends EventEmitter {
74
78
  get indexStore() {
75
79
  return this.index;
76
80
  }
81
+ /**
82
+ * 路径过滤:有 DocumentPort 时文档扩展名不再被 BINARY 短路。
83
+ *
84
+ * @param p - 文件路径
85
+ * @returns 是否跳过
86
+ */
87
+ shouldSkipFile(p) {
88
+ if (this.documentPort && isDocumentPath(p))
89
+ return false;
90
+ return this.adapters.shouldSkipPath(p);
91
+ }
77
92
  /**
78
93
  * 全量/增量索引某源(Phase A)
79
94
  */
@@ -355,7 +370,7 @@ export class KnowledgeIngest extends EventEmitter {
355
370
  if (!filename)
356
371
  return;
357
372
  const abs = join(source.location, filename.toString().split(sep).join(sep));
358
- if (this.adapters.shouldSkipPath(abs))
373
+ if (this.shouldSkipFile(abs))
359
374
  return;
360
375
  this.handleFsChange(sourceId, abs, event === 'rename' ? 'rename' : 'change');
361
376
  });
@@ -654,12 +669,15 @@ export class KnowledgeIngest extends EventEmitter {
654
669
  this.indexVirtualDoc(sourceId, doc);
655
670
  }
656
671
  async parseOne(sourceId, filePath) {
657
- if (this.adapters.shouldSkipPath(filePath)) {
672
+ if (this.shouldSkipFile(filePath)) {
658
673
  this.index.markFileSkipped(sourceId, filePath, 'ignored_path');
659
674
  return false;
660
675
  }
661
676
  const adapter = this.adapters.match(filePath);
662
677
  if (!adapter) {
678
+ if (this.documentPort && isDocumentPath(filePath)) {
679
+ return this.parseDocumentFile(sourceId, filePath);
680
+ }
663
681
  this.index.markFileSkipped(sourceId, filePath, 'no_adapter');
664
682
  return false;
665
683
  }
@@ -700,6 +718,66 @@ export class KnowledgeIngest extends EventEmitter {
700
718
  return false;
701
719
  }
702
720
  }
721
+ /**
722
+ * PDF/Office:DocumentPort 抽取为 Markdown 后走 markdownAdapter 切块。
723
+ * 新鲜度以原件字节 hash 为准。
724
+ *
725
+ * @param sourceId - 知识源 id
726
+ * @param filePath - 本地文件路径
727
+ * @returns 是否成功入索引
728
+ */
729
+ async parseDocumentFile(sourceId, filePath) {
730
+ const port = this.documentPort;
731
+ if (!port)
732
+ return false;
733
+ let st;
734
+ try {
735
+ st = await stat(filePath);
736
+ }
737
+ catch {
738
+ this.index.removeFile(sourceId, filePath);
739
+ return false;
740
+ }
741
+ if (!st.isFile())
742
+ return false;
743
+ if (st.size > this.maxFileBytes) {
744
+ this.index.markFileSkipped(sourceId, filePath, 'oversize');
745
+ return false;
746
+ }
747
+ const raw = await readFile(filePath);
748
+ const contentHash = hashContent(raw);
749
+ if (this.index.isFresh(sourceId, filePath, contentHash)) {
750
+ return true;
751
+ }
752
+ try {
753
+ const result = await port.extract({ path: filePath, name: filePath });
754
+ const markdown = result.markdown?.trim();
755
+ if (!markdown) {
756
+ this.index.markFileSkipped(sourceId, filePath, 'empty_content');
757
+ return false;
758
+ }
759
+ const chunks = markdownAdapter.chunk(markdown, filePath);
760
+ this.index.upsertFile({
761
+ sourceId,
762
+ path: filePath,
763
+ contentHash,
764
+ size: st.size,
765
+ mtime: Math.floor(st.mtimeMs),
766
+ adapterId: `document:${result.backend}`,
767
+ chunks,
768
+ });
769
+ return true;
770
+ }
771
+ catch (err) {
772
+ if (isDocumentExtractError(err)) {
773
+ // 结构化跳过(密码/老格式/超大等),不挡整库
774
+ this.index.markFileSkipped(sourceId, filePath, err.code.toLowerCase());
775
+ return false;
776
+ }
777
+ this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err));
778
+ return false;
779
+ }
780
+ }
703
781
  /**
704
782
  * Phase B:嵌入待处理 chunk(批 + 限速;失败不丢,下次再补)
705
783
  */
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * 附件抽取 — 生成可读伴生文本(.extracted.md)
3
3
  *
4
- * 本期:文本类直接可读(extractPath = path);PDF/Office 无 adapter 不抽取。
4
+ * 文本类直接可读(extractPath = path);PDF/Office 经 DocumentPort 抽取写伴生文件。
5
5
  * 归属:Session 资产侧,不进 Knowledge ingest(要检索再升 source)。
6
6
  */
7
7
  import type { AttachmentKind } from './types.js';
@@ -28,8 +28,9 @@ export declare function extractText(data: Buffer, name: string, kind: Attachment
28
28
  chars: number;
29
29
  } | null;
30
30
  /**
31
- * 伴生抽取文件名(原件旁)
31
+ * 伴生抽取文件名(原件旁);保留源扩展名避免 foo.pdf / foo.docx 撞名
32
32
  *
33
33
  * @param fileName - 原件相对名
34
+ * @returns 例如 report.pdf → report.pdf.extracted.md
34
35
  */
35
36
  export declare function extractCompanionName(fileName: string): string;
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * 附件抽取 — 生成可读伴生文本(.extracted.md)
3
3
  *
4
- * 本期:文本类直接可读(extractPath = path);PDF/Office 无 adapter 不抽取。
4
+ * 文本类直接可读(extractPath = path);PDF/Office 经 DocumentPort 抽取写伴生文件。
5
5
  * 归属:Session 资产侧,不进 Knowledge ingest(要检索再升 source)。
6
6
  */
7
7
  import { extname } from 'node:path';
@@ -28,8 +28,22 @@ export function classifyAttachmentKind(name) {
28
28
  return 'code';
29
29
  if (TEXT_EXTENSIONS.has(ext))
30
30
  return 'text';
31
- if (ext === '.pdf' || ext === '.docx' || ext === '.xlsx' || ext === '.pptx')
31
+ if (ext === '.pdf' ||
32
+ ext === '.docx' ||
33
+ ext === '.xlsx' ||
34
+ ext === '.pptx' ||
35
+ ext === '.doc' ||
36
+ ext === '.xls' ||
37
+ ext === '.ppt' ||
38
+ ext === '.rtf' ||
39
+ ext === '.odt' ||
40
+ ext === '.ods' ||
41
+ ext === '.odp' ||
42
+ ext === '.odg' ||
43
+ ext === '.epub' ||
44
+ ext === '.xmind') {
32
45
  return 'document';
46
+ }
33
47
  return 'other';
34
48
  }
35
49
  /**
@@ -61,12 +75,11 @@ export function extractText(data, name, kind) {
61
75
  return { text, chars: text.length };
62
76
  }
63
77
  /**
64
- * 伴生抽取文件名(原件旁)
78
+ * 伴生抽取文件名(原件旁);保留源扩展名避免 foo.pdf / foo.docx 撞名
65
79
  *
66
80
  * @param fileName - 原件相对名
81
+ * @returns 例如 report.pdf → report.pdf.extracted.md
67
82
  */
68
83
  export function extractCompanionName(fileName) {
69
- const ext = extname(fileName);
70
- const stem = ext ? fileName.slice(0, -ext.length) : fileName;
71
- return `${stem}.extracted.md`;
84
+ return `${fileName}.extracted.md`;
72
85
  }
@@ -6,16 +6,20 @@
6
6
  * 原则:原件 + .manifest.json 为权威;抽取/索引可重建。
7
7
  */
8
8
  import { type AttachmentLimits, type AttachmentUploadInput, type SessionAttachment } from './types.js';
9
+ import type { DocumentPort } from '../../context/capabilities/document/types.js';
9
10
  export interface SessionAttachmentServiceOptions {
10
11
  /** 通常 `OCTOPI_HOME/sessions` */
11
12
  sessionsDir: string;
12
13
  limits?: Partial<AttachmentLimits>;
14
+ /** DocumentPort:PDF/Office 抽取伴生 `.extracted.md`;缺省不抽 */
15
+ documentPort?: DocumentPort | null;
13
16
  }
14
17
  export declare class SessionAttachmentService {
15
18
  private readonly sessionsDir;
16
19
  private readonly limits;
20
+ private readonly documentPort;
17
21
  /**
18
- * @param options - sessionsDir + 可选限额覆盖
22
+ * @param options - sessionsDir + 可选限额覆盖 + documentPort
19
23
  */
20
24
  constructor(options: SessionAttachmentServiceOptions);
21
25
  get limitsConfig(): AttachmentLimits;
@@ -11,16 +11,19 @@ import { mkdir, rename, rm, writeFile } from 'node:fs/promises';
11
11
  import { join } from 'node:path';
12
12
  import { resolveSessionAttachmentsPaths } from './paths.js';
13
13
  import { attachmentExtension, isPathInside, sanitizeAttachmentName } from './sanitize.js';
14
- import { classifyAttachmentKind, extractText, isPlainReadable, } from './extract.js';
14
+ import { classifyAttachmentKind, extractCompanionName, extractText, isPlainReadable, } from './extract.js';
15
15
  import { DEFAULT_ATTACHMENT_LIMITS, } from './types.js';
16
+ import { isDocumentExtractError } from '../../context/capabilities/document/errors.js';
16
17
  export class SessionAttachmentService {
17
18
  sessionsDir;
18
19
  limits;
20
+ documentPort;
19
21
  /**
20
- * @param options - sessionsDir + 可选限额覆盖
22
+ * @param options - sessionsDir + 可选限额覆盖 + documentPort
21
23
  */
22
24
  constructor(options) {
23
25
  this.sessionsDir = options.sessionsDir;
26
+ this.documentPort = options.documentPort ?? null;
24
27
  // 只合并已定义字段:`{ ...defaults, allowedExtensions: undefined }` 会把默认值抹掉
25
28
  const o = options.limits ?? {};
26
29
  this.limits = {
@@ -134,9 +137,48 @@ export class SessionAttachmentService {
134
137
  item.parse = { ok: false, error: 'binary content in text extension' };
135
138
  }
136
139
  }
140
+ else if (kind === 'document' && this.documentPort) {
141
+ const companion = extractCompanionName(uniqueName);
142
+ item.extractPath = companion;
143
+ try {
144
+ const result = await this.documentPort.extract({
145
+ data: new Uint8Array(data),
146
+ name: uniqueName,
147
+ });
148
+ const absExtract = join(root, companion);
149
+ if (!isPathInside(root, absExtract)) {
150
+ throw new Error('invalid extract path');
151
+ }
152
+ await writeFile(absExtract, result.markdown);
153
+ item.status = 'parsed';
154
+ item.parse = { ok: true, chars: result.markdown.length };
155
+ }
156
+ catch (err) {
157
+ item.status = 'parse_failed';
158
+ if (isDocumentExtractError(err)) {
159
+ item.parse = {
160
+ ok: false,
161
+ error: `${err.code}${err.requires ? ` (requires: ${err.requires.join(',')})` : ''}`,
162
+ };
163
+ }
164
+ else {
165
+ item.parse = {
166
+ ok: false,
167
+ error: err instanceof Error ? err.message : String(err),
168
+ };
169
+ }
170
+ }
171
+ }
137
172
  else if (kind === 'image') {
138
173
  item.status = 'ready';
139
174
  }
175
+ else if (kind === 'document') {
176
+ item.status = 'ready';
177
+ item.parse = {
178
+ ok: false,
179
+ error: 'document extract disabled (no DocumentPort)',
180
+ };
181
+ }
140
182
  else {
141
183
  item.status = 'ready';
142
184
  item.parse = { ok: false, error: 'no extractor for this type (see OP adapter)' };
package/dist/index.d.ts CHANGED
@@ -71,8 +71,8 @@ export type { AgentPersona, ModelConfig, AgentDefinition } from './harness/share
71
71
  export { DefaultToolBus } from './harness/extension/plugin-ecosystem/tools/tool-bus.js';
72
72
  export { getBuiltinTools, createShellTool, createFileReadTool, createFileWriteTool, createFileListTool } from './harness/extension/plugin-ecosystem/tools/builtin.js';
73
73
  export { createToolSet } from './harness/extension/plugin-ecosystem/tools/tool-set.js';
74
- export { createSummaryPort, applyToolOutputGate, createCompactEngine } from './harness/context/capabilities/index.js';
75
- export type { SummaryPort, SummaryPolicy, ContentUnit, ToolSummarySupport, CompactEngine } from './harness/context/capabilities/index.js';
74
+ export { createSummaryPort, applyToolOutputGate, createCompactEngine, createDefaultDocumentPort, DocumentExtractError } from './harness/context/capabilities/index.js';
75
+ export type { SummaryPort, SummaryPolicy, ContentUnit, ToolSummarySupport, CompactEngine, DocumentPort, ExtractResult, ExtractSource } from './harness/context/capabilities/index.js';
76
76
  export type { ToolSet, ToolSetConfig } from './harness/extension/plugin-ecosystem/tools/tool-set.js';
77
77
  export { createWebSearchTool } from './harness/extension/plugin-ecosystem/tools/web-search.js';
78
78
  export type { WebSearchToolOptions } from './harness/extension/plugin-ecosystem/tools/web-search.js';
package/dist/index.js CHANGED
@@ -47,7 +47,7 @@ export { DefaultSkillManager, FileSystemSkillSource } from './harness/extension/
47
47
  export { DefaultToolBus } from './harness/extension/plugin-ecosystem/tools/tool-bus.js';
48
48
  export { getBuiltinTools, createShellTool, createFileReadTool, createFileWriteTool, createFileListTool } from './harness/extension/plugin-ecosystem/tools/builtin.js';
49
49
  export { createToolSet } from './harness/extension/plugin-ecosystem/tools/tool-set.js';
50
- export { createSummaryPort, applyToolOutputGate, createCompactEngine } from './harness/context/capabilities/index.js';
50
+ export { createSummaryPort, applyToolOutputGate, createCompactEngine, createDefaultDocumentPort, DocumentExtractError } from './harness/context/capabilities/index.js';
51
51
  export { createWebSearchTool } from './harness/extension/plugin-ecosystem/tools/web-search.js';
52
52
  export { createSessionHistoryPort, resolveHistoryAccess, } from './harness/session/history/index.js';
53
53
  export { createSessionHistoryTools, createSessionSearchTool, createSessionReadTool, } from './harness/extension/plugin-ecosystem/tools/session-history.js';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@octopi-agent/engine",
3
- "version": "0.55.0",
3
+ "version": "0.56.0",
4
4
  "description": "Octopi Engine — Harness product domains, library integrations, and embeddable Agent.run()",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -80,12 +80,17 @@
80
80
  },
81
81
  "dependencies": {
82
82
  "@modelcontextprotocol/sdk": "^1.29.0",
83
- "@octopi-agent/core": "^0.55.0",
83
+ "@octopi-agent/core": "0.56.0",
84
84
  "js-yaml": "^4.3.2",
85
85
  "zod": "^4.4.3"
86
86
  },
87
87
  "optionalDependencies": {
88
- "sqlite-vec": "^0.1.9"
88
+ "fflate": "^0.8.3",
89
+ "mammoth": "^1.13.0",
90
+ "officeparser": "^8.1.0",
91
+ "sqlite-vec": "^0.1.9",
92
+ "unpdf": "^1.8.1",
93
+ "xlsx": "^0.18.5"
89
94
  },
90
95
  "private": false,
91
96
  "publishConfig": {