@octopi-agent/engine 0.56.0 → 0.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/dist/config-schema/context.d.ts +1 -1
  2. package/dist/config-schema/context.js +1 -1
  3. package/dist/config-schema/engine.d.ts +43 -1
  4. package/dist/config-schema/substrate.d.ts +42 -0
  5. package/dist/config-schema/substrate.js +58 -0
  6. package/dist/config.d.ts +39 -1
  7. package/dist/harness/agent/builder.d.ts +14 -0
  8. package/dist/harness/agent/builder.js +14 -2
  9. package/dist/harness/agent/config-bridge.js +3 -0
  10. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
  11. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
  12. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
  13. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
  14. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
  15. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
  16. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
  17. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
  18. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
  19. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
  20. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
  21. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
  22. package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
  23. package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
  24. package/dist/harness/capabilities/document/factory.d.ts +33 -0
  25. package/dist/harness/capabilities/document/factory.js +37 -0
  26. package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
  27. package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
  28. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
  29. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
  30. package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
  31. package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
  32. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
  33. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
  34. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
  35. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
  36. package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
  37. package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
  38. package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
  39. package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
  40. package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
  41. package/dist/harness/capabilities/document/worker-port.js +78 -0
  42. package/dist/harness/context/capabilities/index.d.ts +2 -2
  43. package/dist/harness/context/capabilities/index.js +2 -2
  44. package/dist/harness/context/constitution/default-agents.md +1 -0
  45. package/dist/harness/context/index.d.ts +2 -2
  46. package/dist/harness/context/index.js +1 -1
  47. package/dist/harness/context/layer-types.d.ts +26 -2
  48. package/dist/harness/context/layer-types.js +59 -11
  49. package/dist/harness/context/layers.d.ts +3 -0
  50. package/dist/harness/context/layers.js +38 -12
  51. package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
  52. package/dist/harness/context/system-prompt-assembler.js +2 -0
  53. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
  54. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +115 -5
  55. package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
  56. package/dist/harness/index.d.ts +5 -3
  57. package/dist/harness/index.js +3 -2
  58. package/dist/harness/knowledge/adapters.d.ts +8 -5
  59. package/dist/harness/knowledge/adapters.js +53 -13
  60. package/dist/harness/knowledge/catalog-types.d.ts +4 -0
  61. package/dist/harness/knowledge/client.d.ts +197 -0
  62. package/dist/harness/knowledge/client.js +237 -0
  63. package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
  64. package/dist/harness/knowledge/connectors.d.ts +3 -3
  65. package/dist/harness/knowledge/connectors.js +14 -8
  66. package/dist/harness/knowledge/db.d.ts +42 -3
  67. package/dist/harness/knowledge/db.js +466 -83
  68. package/dist/harness/knowledge/describe.d.ts +2 -0
  69. package/dist/harness/knowledge/describe.js +5 -1
  70. package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
  71. package/dist/harness/knowledge/document-extract-worker.js +68 -0
  72. package/dist/harness/knowledge/embed-runner.d.ts +53 -0
  73. package/dist/harness/knowledge/embed-runner.js +171 -0
  74. package/dist/harness/knowledge/engine-thread.d.ts +1 -0
  75. package/dist/harness/knowledge/engine-thread.js +302 -0
  76. package/dist/harness/knowledge/extract-document.d.ts +33 -0
  77. package/dist/harness/knowledge/extract-document.js +86 -0
  78. package/dist/harness/knowledge/fetchers.d.ts +14 -3
  79. package/dist/harness/knowledge/fetchers.js +52 -16
  80. package/dist/harness/knowledge/file-identity.d.ts +53 -0
  81. package/dist/harness/knowledge/file-identity.js +172 -0
  82. package/dist/harness/knowledge/file-index-store.d.ts +50 -0
  83. package/dist/harness/knowledge/file-index-store.js +132 -0
  84. package/dist/harness/knowledge/file-limits.d.ts +86 -0
  85. package/dist/harness/knowledge/file-limits.js +175 -0
  86. package/dist/harness/knowledge/fts.d.ts +60 -0
  87. package/dist/harness/knowledge/fts.js +218 -0
  88. package/dist/harness/knowledge/grounding.d.ts +3 -2
  89. package/dist/harness/knowledge/http-app.d.ts +50 -0
  90. package/dist/harness/knowledge/http-app.js +652 -0
  91. package/dist/harness/knowledge/http-bridge.d.ts +49 -0
  92. package/dist/harness/knowledge/http-bridge.js +17 -0
  93. package/dist/harness/knowledge/index-store.d.ts +166 -63
  94. package/dist/harness/knowledge/index-store.js +1025 -282
  95. package/dist/harness/knowledge/index.d.ts +47 -5
  96. package/dist/harness/knowledge/index.js +24 -2
  97. package/dist/harness/knowledge/ingest.d.ts +322 -49
  98. package/dist/harness/knowledge/ingest.js +1383 -267
  99. package/dist/harness/knowledge/job-control-state.d.ts +29 -0
  100. package/dist/harness/knowledge/job-control-state.js +44 -0
  101. package/dist/harness/knowledge/job-control.d.ts +35 -0
  102. package/dist/harness/knowledge/job-control.js +86 -0
  103. package/dist/harness/knowledge/job-queue.d.ts +62 -0
  104. package/dist/harness/knowledge/job-queue.js +173 -0
  105. package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
  106. package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
  107. package/dist/harness/knowledge/local-stack.d.ts +27 -0
  108. package/dist/harness/knowledge/local-stack.js +63 -0
  109. package/dist/harness/knowledge/membership-store.d.ts +65 -0
  110. package/dist/harness/knowledge/membership-store.js +198 -0
  111. package/dist/harness/knowledge/network-guard.d.ts +11 -2
  112. package/dist/harness/knowledge/network-guard.js +130 -85
  113. package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
  114. package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
  115. package/dist/harness/knowledge/query-service.d.ts +272 -0
  116. package/dist/harness/knowledge/query-service.js +326 -0
  117. package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
  118. package/dist/harness/knowledge/query-worker-client.js +207 -0
  119. package/dist/harness/knowledge/query-worker.d.ts +1 -0
  120. package/dist/harness/knowledge/query-worker.js +186 -0
  121. package/dist/harness/knowledge/read-http.d.ts +35 -0
  122. package/dist/harness/knowledge/read-http.js +373 -0
  123. package/dist/harness/knowledge/retriever.d.ts +30 -5
  124. package/dist/harness/knowledge/retriever.js +98 -35
  125. package/dist/harness/knowledge/secret-scan.d.ts +13 -2
  126. package/dist/harness/knowledge/secret-scan.js +25 -2
  127. package/dist/harness/knowledge/serve.d.ts +48 -0
  128. package/dist/harness/knowledge/serve.js +336 -0
  129. package/dist/harness/knowledge/source-store.d.ts +83 -16
  130. package/dist/harness/knowledge/source-store.js +205 -78
  131. package/dist/harness/knowledge/start-service-process.d.ts +49 -0
  132. package/dist/harness/knowledge/start-service-process.js +158 -0
  133. package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
  134. package/dist/harness/knowledge/text-parse-worker.js +54 -0
  135. package/dist/harness/knowledge/topics.d.ts +13 -0
  136. package/dist/harness/knowledge/topics.js +48 -0
  137. package/dist/harness/knowledge/types.d.ts +1 -1
  138. package/dist/harness/knowledge/types.js +1 -1
  139. package/dist/harness/knowledge/vector-ann.d.ts +42 -0
  140. package/dist/harness/knowledge/vector-ann.js +68 -0
  141. package/dist/harness/knowledge/writer-lock.d.ts +13 -0
  142. package/dist/harness/knowledge/writer-lock.js +71 -0
  143. package/dist/harness/knowledge/writer-service.d.ts +194 -0
  144. package/dist/harness/knowledge/writer-service.js +355 -0
  145. package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
  146. package/dist/harness/knowledge/writer-worker-client.js +381 -0
  147. package/dist/harness/knowledge/writer-worker.d.ts +1 -0
  148. package/dist/harness/knowledge/writer-worker.js +189 -0
  149. package/dist/harness/memory/index.d.ts +3 -2
  150. package/dist/harness/memory/index.js +3 -1
  151. package/dist/harness/memory/retrieval-rank.d.ts +51 -0
  152. package/dist/harness/memory/retrieval-rank.js +60 -0
  153. package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
  154. package/dist/harness/memory/sqlite/embedding.js +109 -16
  155. package/dist/harness/memory/sqlite/index.d.ts +2 -2
  156. package/dist/harness/memory/sqlite/index.js +1 -1
  157. package/dist/harness/memory/sqlite/keyword-search.d.ts +41 -2
  158. package/dist/harness/memory/sqlite/keyword-search.js +104 -17
  159. package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
  160. package/dist/harness/memory/sqlite/memory-store.js +89 -39
  161. package/dist/harness/memory/store.js +39 -21
  162. package/dist/harness/memory/types.d.ts +7 -0
  163. package/dist/harness/run/runner.d.ts +1 -0
  164. package/dist/harness/run/runner.js +4 -2
  165. package/dist/harness/session/attachments/service.d.ts +1 -1
  166. package/dist/harness/session/attachments/service.js +1 -1
  167. package/dist/harness/session/tasks/index.d.ts +1 -1
  168. package/dist/harness/session/tasks/index.js +1 -1
  169. package/dist/harness/session/tasks/render.d.ts +10 -0
  170. package/dist/harness/session/tasks/render.js +27 -0
  171. package/dist/index.d.ts +4 -2
  172. package/dist/index.js +2 -1
  173. package/dist/worker-path.d.ts +8 -0
  174. package/dist/worker-path.js +15 -0
  175. package/package.json +2 -2
@@ -6,23 +6,62 @@
6
6
  */
7
7
  export { KnowledgeDatabase } from './db.js';
8
8
  export type { KnowledgeDatabaseOptions } from './db.js';
9
+ export { identifyLocalFile, identifyUrl, identifyConnector, normalizeFsPath, normalizePathLexical, urlIdentityKey, connectorIdentityKey, logicalPathFrom, isPathPrefix, canonicalizeUrl, } from './file-identity.js';
10
+ export type { FileIdentity } from './file-identity.js';
11
+ export { MembershipStore } from './membership-store.js';
12
+ export type { MembershipRow, FileRow, ReconcileResult } from './membership-store.js';
13
+ export { FileIndexStore } from './file-index-store.js';
14
+ export type { ChunkDraft, UpsertFileResult } from './file-index-store.js';
9
15
  export { KnowledgeSourceStore } from './source-store.js';
10
16
  export { KnowledgeIndexStore, hashContent } from './index-store.js';
11
17
  export type { IndexedFileRecord, ChunkHit } from './index-store.js';
18
+ export { KnowledgeFts, buildFtsTokens, buildFtsQuery } from './fts.js';
19
+ export { vectorBucket, queryBuckets, scoreAnnCandidates, VECTOR_BUCKETS, } from './vector-ann.js';
12
20
  export { FormatAdapterRegistry, textAdapter, markdownAdapter, htmlAdapter, codeAdapter, chunkCodeBySymbols, IGNORED_DIRS, BINARY_EXTENSIONS, } from './adapters.js';
13
21
  export type { FormatAdapter, KnowledgeChunkDraft } from './adapters.js';
14
22
  export { KnowledgeIngest } from './ingest.js';
15
- export type { KnowledgeIngestOptions, IngestProgressEvent, IngestJobKind } from './ingest.js';
23
+ export type { KnowledgeIngestOptions, IngestProgressEvent, IngestJobKind, } from './ingest.js';
24
+ export { KnowledgeJobControl } from './job-control.js';
25
+ export { JobQueue } from './job-queue.js';
26
+ export type { IngestJobKind as JobKind, JobRow } from './job-queue.js';
27
+ export { EmbedRunner } from './embed-runner.js';
28
+ export type { EmbedSecretPolicy } from './embed-runner.js';
29
+ export { DEFAULT_KNOWLEDGE_FILE_LIMITS, classifyFileKind, decideBySize, formatMaxBytes, isRetryableSkipReason, parseTimeoutForSize, resolveKnowledgeFileLimits, } from './file-limits.js';
30
+ export type { KnowledgeFileKind, KnowledgeFileLimits, KnowledgeFileLimitsInput, OversizePolicy, } from './file-limits.js';
16
31
  export { LocalFsFetcher, UrlFetcher, } from './fetchers.js';
17
- export type { SourceFetcher, DiscoveredDocRef, VirtualDocument, } from './fetchers.js';
32
+ export type { SourceFetcher, DiscoveredDocRef, DiscoverResult, VirtualDocument, } from './fetchers.js';
18
33
  export { ConnectorRegistry, RestConnector } from './connectors.js';
19
34
  export type { KnowledgeConnector, ConnectorContext, RestConnectorConfig } from './connectors.js';
20
35
  export { ConnectorFetcher } from './connector-fetcher.js';
21
- export { assertUrlAllowed, guardedFetch, isRestrictedIp } from './network-guard.js';
36
+ export { assertUrlAllowed, authHeadersForUrl, guardedFetch, isRestrictedIp } from './network-guard.js';
22
37
  export type { NetworkGuardOptions, FetchResult } from './network-guard.js';
23
38
  export { htmlToStructuredText, looksLikeHtml } from './html.js';
24
39
  export { KnowledgeRetriever } from './retriever.js';
25
- export type { HybridSearchOptions, HybridSearchResult, AutoGroundDecision, GroundingMode, KnowledgeRecallMode, KnowledgeRetrieverOptions, } from './retriever.js';
40
+ export { LocalKnowledgeQueryService, createKnowledgeQueryService, } from './query-service.js';
41
+ export type { KnowledgeQueryService, SearchQuery, AutoGroundQuery, ListSourcesQuery, QueryIdentity, PrincipalStats, ReadChunkResult, SourceStatsBundle, SourceDetail, ListJobsQuery, ReadySnapshot, QueryWorkerRole, } from './query-service.js';
42
+ export { readJobControlState } from './job-control-state.js';
43
+ export type { KnowledgeJobControlState } from './job-control-state.js';
44
+ export { handleKnowledgeReadHttp, isPureReadRoute, authenticateRead, } from './read-http.js';
45
+ export type { ReadHttpDeps } from './read-http.js';
46
+ export { KnowledgeHttpApp, createKnowledgeHttpApp } from './http-app.js';
47
+ export type { KnowledgeServiceToken, KnowledgeServiceOptions, AuthContext } from './http-app.js';
48
+ export { LocalKnowledgeWriteService, } from './writer-service.js';
49
+ export type { KnowledgeWriteService, WriteIdentity, WriteAbortStats, WriteResumeStats, WriteRegisterResult, WriteDescribeResult, WriteReprocessResult, } from './writer-service.js';
50
+ export { WorkerWriteService } from './writer-worker-client.js';
51
+ export type { WorkerWriteStartOptions } from './writer-worker-client.js';
52
+ export { createLocalKnowledgeStack } from './local-stack.js';
53
+ export type { LocalKnowledgeStack } from './local-stack.js';
54
+ export { startKnowledgeService } from './serve.js';
55
+ export type { KnowledgeServeOptions, KnowledgeServeHandle } from './serve.js';
56
+ export { startKnowledgeServiceProcess } from './start-service-process.js';
57
+ export type { KnowledgeServiceProcessOptions, KnowledgeServiceProcessHandle, } from './start-service-process.js';
58
+ export { acquireKnowledgeWriterLock } from './writer-lock.js';
59
+ export type { WriterLockHolder } from './writer-lock.js';
60
+ export { parseTextInWorker } from './parse-text-in-worker.js';
61
+ export type { TextParseResult, TextParseWorkerOptions } from './parse-text-in-worker.js';
62
+ export { KnowledgeClient, ClientKnowledgeGrounding } from './client.js';
63
+ export type { KnowledgeClientOptions } from './client.js';
64
+ export type { HybridSearchOptions, HybridSearchResult, AutoGroundDecision, AutoGroundPort, GroundingMode, KnowledgeRecallMode, KnowledgeRetrieverOptions, } from './retriever.js';
26
65
  export { GroundingAssembler, formatKnowledgeGroundingMessage, resolveGroundingQuery, wrapUntrustedKnowledgeBlock, isKnowledgeGroundingMessage, stripKnowledgeGrounding, KNOWLEDGE_GROUNDING_SOURCE, } from './grounding.js';
27
66
  export type { GroundingPack, GroundingAssemblerOptions } from './grounding.js';
28
67
  export { KnowledgeHitLog } from './hit-log.js';
@@ -31,8 +70,11 @@ export { KnowledgePurger } from './purge.js';
31
70
  export type { PurgeResult } from './purge.js';
32
71
  export { resolveKnowledgePaths } from './paths.js';
33
72
  export type { KnowledgePaths } from './paths.js';
73
+ export { deriveTopicsFromPaths } from './topics.js';
74
+ export type { KnowledgeCatalogItem, KnowledgeCatalogProvider } from './catalog-types.js';
34
75
  export { generateKnowledgeDescription, heuristicDescription, } from './describe.js';
35
76
  export type { KnowledgeDescribePort, KnowledgeDescribeOptions, KnowledgeDescribeResult, } from './describe.js';
36
- export { scanSecretShapes } from './secret-scan.js';
77
+ export { scanSecretShapes, redactSecretShapes } from './secret-scan.js';
78
+ export type { RedactResult } from './secret-scan.js';
37
79
  export type { KnowledgeSource, KnowledgeSourceInput, KnowledgeSourcePatch, KnowledgeSourceKind, KnowledgeSourceStatus, KnowledgeSourceSync, KnowledgeSourceError, KnowledgeSourceNetwork, KnowledgeSourceDiscover, KnowledgeScopeRef, KnowledgeScopeLevel, KnowledgeSourceId, KnowledgeChunkId, KnowledgeVisibilityTargetType, KnowledgeVisibilityOp, KnowledgeSessionVisibilityItem, KnowledgeSessionVisibilityInput, Branded, } from './types.js';
38
80
  export { asSourceId, asChunkId } from './types.js';
@@ -5,20 +5,42 @@
5
5
  * 规格:arch/knowledge-layer.md
6
6
  */
7
7
  export { KnowledgeDatabase } from './db.js';
8
+ export { identifyLocalFile, identifyUrl, identifyConnector, normalizeFsPath, normalizePathLexical, urlIdentityKey, connectorIdentityKey, logicalPathFrom, isPathPrefix, canonicalizeUrl, } from './file-identity.js';
9
+ export { MembershipStore } from './membership-store.js';
10
+ export { FileIndexStore } from './file-index-store.js';
8
11
  export { KnowledgeSourceStore } from './source-store.js';
9
12
  export { KnowledgeIndexStore, hashContent } from './index-store.js';
13
+ export { KnowledgeFts, buildFtsTokens, buildFtsQuery } from './fts.js';
14
+ export { vectorBucket, queryBuckets, scoreAnnCandidates, VECTOR_BUCKETS, } from './vector-ann.js';
10
15
  export { FormatAdapterRegistry, textAdapter, markdownAdapter, htmlAdapter, codeAdapter, chunkCodeBySymbols, IGNORED_DIRS, BINARY_EXTENSIONS, } from './adapters.js';
11
16
  export { KnowledgeIngest } from './ingest.js';
17
+ export { KnowledgeJobControl } from './job-control.js';
18
+ export { JobQueue } from './job-queue.js';
19
+ export { EmbedRunner } from './embed-runner.js';
20
+ export { DEFAULT_KNOWLEDGE_FILE_LIMITS, classifyFileKind, decideBySize, formatMaxBytes, isRetryableSkipReason, parseTimeoutForSize, resolveKnowledgeFileLimits, } from './file-limits.js';
12
21
  export { LocalFsFetcher, UrlFetcher, } from './fetchers.js';
13
22
  export { ConnectorRegistry, RestConnector } from './connectors.js';
14
23
  export { ConnectorFetcher } from './connector-fetcher.js';
15
- export { assertUrlAllowed, guardedFetch, isRestrictedIp } from './network-guard.js';
24
+ export { assertUrlAllowed, authHeadersForUrl, guardedFetch, isRestrictedIp } from './network-guard.js';
16
25
  export { htmlToStructuredText, looksLikeHtml } from './html.js';
17
26
  export { KnowledgeRetriever } from './retriever.js';
27
+ export { LocalKnowledgeQueryService, createKnowledgeQueryService, } from './query-service.js';
28
+ export { readJobControlState } from './job-control-state.js';
29
+ export { handleKnowledgeReadHttp, isPureReadRoute, authenticateRead, } from './read-http.js';
30
+ export { KnowledgeHttpApp, createKnowledgeHttpApp } from './http-app.js';
31
+ export { LocalKnowledgeWriteService, } from './writer-service.js';
32
+ export { WorkerWriteService } from './writer-worker-client.js';
33
+ export { createLocalKnowledgeStack } from './local-stack.js';
34
+ export { startKnowledgeService } from './serve.js';
35
+ export { startKnowledgeServiceProcess } from './start-service-process.js';
36
+ export { acquireKnowledgeWriterLock } from './writer-lock.js';
37
+ export { parseTextInWorker } from './parse-text-in-worker.js';
38
+ export { KnowledgeClient, ClientKnowledgeGrounding } from './client.js';
18
39
  export { GroundingAssembler, formatKnowledgeGroundingMessage, resolveGroundingQuery, wrapUntrustedKnowledgeBlock, isKnowledgeGroundingMessage, stripKnowledgeGrounding, KNOWLEDGE_GROUNDING_SOURCE, } from './grounding.js';
19
40
  export { KnowledgeHitLog } from './hit-log.js';
20
41
  export { KnowledgePurger } from './purge.js';
21
42
  export { resolveKnowledgePaths } from './paths.js';
43
+ export { deriveTopicsFromPaths } from './topics.js';
22
44
  export { generateKnowledgeDescription, heuristicDescription, } from './describe.js';
23
- export { scanSecretShapes } from './secret-scan.js';
45
+ export { scanSecretShapes, redactSecretShapes } from './secret-scan.js';
24
46
  export { asSourceId, asChunkId } from './types.js';
@@ -1,7 +1,7 @@
1
1
  /**
2
- * KnowledgeIngest — 解析/切块/索引队列(P2 Phase A)
2
+ * KnowledgeIngest ?解析/切块/索引队列(P2 Phase A?
3
3
  *
4
- * 原则:ingest 与对话并发,不作 turn 同步前置;队列背压不丢任务;source 级锁。
4
+ * 原则:ingest 话并发,不作 turn 前置;队列背压不务;source 级锁?
5
5
  */
6
6
  import { EventEmitter } from 'node:events';
7
7
  import { FormatAdapterRegistry } from './adapters.js';
@@ -11,63 +11,105 @@ import { type SourceFetcher, type VirtualDocument } from './fetchers.js';
11
11
  import { ConnectorRegistry } from './connectors.js';
12
12
  import type { CredentialStore } from '../governance/credentials/store.js';
13
13
  import type { EmbeddingProvider } from '../memory/sqlite/embedding.js';
14
- import type { DocumentPort } from '../context/capabilities/document/types.js';
15
- export type IngestJobKind = 'parse_file' | 'walk_source' | 'drop_file' | 'embed_source' | 'fetch_doc';
14
+ import type { DocumentPort } from '../capabilities/document/types.js';
15
+ import { type KnowledgeFileLimitsInput } from './file-limits.js';
16
+ import { type EmbedSecretPolicy } from './embed-runner.js';
17
+ /**
18
+ * 流式 SHA-256(可中止):禁止在 Engine 事件循环 `readFile` 整文件再 hash。
19
+ *
20
+ * @param filePath - 本地文件
21
+ * @param signal - 中止信号
22
+ * @returns hex digest
23
+ */
24
+ export declare function hashFileStreaming(filePath: string, signal?: AbortSignal): Promise<string>;
25
+ import { type IngestJobKind as JobKind } from './job-queue.js';
26
+ import { type KnowledgeJobControlState } from './job-control-state.js';
27
+ export type { EmbedSecretPolicy };
28
+ export type IngestJobKind = JobKind;
29
+ export type { KnowledgeJobControlState };
16
30
  export interface IngestProgressEvent {
17
31
  type: 'job' | 'source' | 'error' | 'embed';
18
32
  sourceId: string;
19
33
  path?: string;
20
34
  status: string;
21
35
  detail?: string;
36
+ /** SSE 过滤用:源归属(Writer 在 emit 时补齐,API 不得再 RPC 查) */
37
+ tenantId?: string;
38
+ registeredBy?: string;
39
+ visibility?: string;
22
40
  }
23
41
  export interface KnowledgeIngestOptions {
24
42
  sourceStore: KnowledgeSourceStore;
25
43
  indexStore?: KnowledgeIndexStore;
26
44
  adapterRegistry?: FormatAdapterRegistry;
27
- /** parse 并发(默认 4) */
45
+ /** parse 并发(默认 8) */
28
46
  parseConcurrency?: number;
29
- /** 队列深度上限(超出只报警,不丢任务) */
47
+ /** 文档(PDF/Office)抽取并发(默认 2;大文件抢占 CPU/内存) */
48
+ documentParseConcurrency?: number;
49
+ /** 队列深度上限(超出只,不务) */
30
50
  maxQueueDepth?: number;
31
- /** fs watch debounce ms(默认 2000) */
51
+ /** fs watch debounce ms(默?2000?*/
32
52
  debounceMs?: number;
33
- /** 单文件大小上限(默认 5MB;超出 skip) */
53
+ /** 单文件大小兜底上限(格式分级?fileLimits?*/
34
54
  maxFileBytes?: number;
35
- /** Phase B embedding(未配则纯关键词) */
55
+ /**
56
+ * 按格式分级的文件 + 部分抽取策略?
57
+ * 机器资源不足时下?maxBytes / hardMaxFileBytes?
58
+ */
59
+ fileLimits?: KnowledgeFileLimitsInput;
60
+ /** 单文?parse 超时 ms(默?45000?*/
61
+ parseTimeoutMs?: number;
62
+ /** Phase B embedding(未配则?*/
36
63
  embeddingProvider?: EmbeddingProvider | null;
37
- /** embed 批大小(默认 32) */
64
+ /** embed 批大小( 32?*/
38
65
  embedBatch?: number;
39
- /** embed 最小间隔 ms(限速,默认 0) */
66
+ /** embed 小间?ms(限速, 0?*/
40
67
  embedMinIntervalMs?: number;
41
- /** embed 并发(默认复用 parseConcurrency) */
68
+ /** embed 并发(默?parseConcurrency?*/
42
69
  embedConcurrency?: number;
43
- /** 磁盘水位告警(默认 false;仅告警不丢任务) */
70
+ /** 磁盘水位(默?false;仅不丢任务?*/
44
71
  diskWatermarkAlert?: boolean;
45
- /** 凭证库(source.authRef 解析;可选) */
72
+ /** 库(source.authRef 解析;可选) */
46
73
  credentials?: CredentialStore | null;
47
- /** 自定义 fetcher 覆盖(测试/扩展) */
74
+ /** ?fetcher 覆盖(测?扩展?*/
48
75
  fetchers?: Partial<Record<'local' | 'url' | 'connector', SourceFetcher>>;
49
- /** connector 注册表(默认含 rest) */
76
+ /** connector 注册?rest?*/
50
77
  connectors?: ConnectorRegistry;
51
- /** poll 调度 tick ms(默认 60s) */
78
+ /** poll 调度 tick ms(默?60s?*/
52
79
  pollTickMs?: number;
53
- /** 单源最小 poll 间隔 ms(默认 15min) */
80
+ /** 单源?poll 间隔 ms(默?15min?*/
54
81
  pollMinIntervalMs?: number;
55
- /** 每轮 poll 最多处理源数(成本上限,默认 8) */
82
+ /** 每轮 poll 理源数(成本上限,默?8?*/
56
83
  maxPollPerTick?: number;
57
84
  /**
58
- * DocumentPort:启用后 PDF/Office 等走抽取 → Markdown 切块
59
- * (缺省 null = 维持 BINARY skip 语义)
85
+ * DocumentPort:启用后 PDF/Office 等走抽取 ?Markdown 切块
86
+ * (缺?null = 维持 BINARY skip ?
60
87
  */
61
88
  documentPort?: DocumentPort | null;
89
+ /**
90
+ * documents.* 配置 — 与 Gateway 同源;worker 抽取与 legacy/soffice 共用
91
+ */
92
+ documentConfig?: import('../capabilities/document/factory.js').DocumentCapabilityConfig | null;
93
+ /**
94
+ * embedding 外发前敏感形态策略( redact?
95
+ * - allow:原发(内网/模型?
96
+ * - redact:命 [REDACTED:rule] ?embed
97
+ * - skip:命不写向量(关仍可搜)
98
+ */
99
+ embedSecretPolicy?: EmbedSecretPolicy;
62
100
  }
63
101
  export declare class KnowledgeIngest extends EventEmitter {
64
102
  private readonly sources;
65
103
  private readonly index;
66
104
  private readonly adapters;
67
105
  private readonly parseConcurrency;
106
+ private readonly documentParseConcurrency;
68
107
  private readonly maxQueueDepth;
69
108
  private readonly debounceMs;
70
109
  private readonly maxFileBytes;
110
+ private readonly fileLimits;
111
+ /** 单文?parse/fetch 超时(默?45s;大 Office 抽取会堵事件?*/
112
+ private readonly parseTimeoutMs;
71
113
  private readonly embedding;
72
114
  private readonly embedBatch;
73
115
  private readonly embedMinIntervalMs;
@@ -81,105 +123,336 @@ export declare class KnowledgeIngest extends EventEmitter {
81
123
  private readonly pollMinIntervalMs;
82
124
  private readonly maxPollPerTick;
83
125
  private readonly documentPort;
84
- private lastEmbedAt;
126
+ private readonly documentConfig;
127
+ private readonly embedSecretPolicy;
128
+ private readonly embedRunner;
85
129
  private pollTimer;
86
- private running;
130
+ private reconcileTimer;
131
+ /** 源级:DB 权威 + ?AbortSignal(跨重启?*/
132
+ private readonly jobControl;
133
+ private readonly jobQueue;
134
+ /** 源任务稳定(?queued/running)时回调 ?用于 auto-describe 等收?*/
135
+ onSourceSettled?: (sourceId: string) => void;
136
+ /** parse 缺口时间 */
137
+ private lastParseGapScanAt;
138
+ /** watch 突发后要求尽快缺口补扫(覆盖 60s 节流) */
139
+ private gapScanUrgent;
140
+ private gapScanTimer;
141
+ private runningParse;
142
+ private runningDocParse;
87
143
  private runningEmbed;
88
144
  private draining;
89
- /** source 级锁:同一 source 不并行 walk/parse 冲突写 */
145
+ private kickAgain;
146
+ /** source 级锁:同 source 不并?walk/parse 冲突?*/
90
147
  private sourceLocks;
91
- /** walk 发现的文件数(Phase A coverage 分母) */
148
+ /** walk 发现的文件数(Phase A coverage 分母?*/
92
149
  private discoveredFiles;
93
150
  private watchers;
94
151
  private debounceTimers;
95
152
  private disposed;
96
153
  constructor(options: KnowledgeIngestOptions);
154
+ private lastCleanupAt;
155
+ /**
156
+ * 清掉「非?knowledge_files (目录曾 no_adapter?
157
+ *
158
+ * ?skipped/error 且无 chunk :有 chunk 的仍?pruneMissing/磁盘对齐?
159
+ * 60s 节流,避免每?reconcile ?stat?
160
+ *
161
+ * @param sourceId - 限定源;省略则全部源
162
+ * @param opts.force - 忽略节流
163
+ * @returns 删除行数
164
+ */
165
+ cleanupNonFileIndexRows(sourceId?: string, opts?: {
166
+ force?: boolean;
167
+ }): number;
168
+ /**
169
+ * 无跨进程 Lease:本进程时把历史 running 收回 queued?
170
+ * 仅构造时调用行中同进程的 running 法的?
171
+ */
172
+ private reclaimOrphanRunningJobs;
97
173
  get indexStore(): KnowledgeIndexStore;
98
174
  /**
99
- * 路径过滤:有 DocumentPort 时文档扩展名不再被 BINARY 短路。
175
+ * 过滤:文档扩展名交给 worker 抽取,不 BINARY ?
100
176
  *
101
- * @param p - 文件路径
102
- * @returns 是否跳过
177
+ * @param p - 文件
178
+ * @returns 跳过
103
179
  */
104
180
  private shouldSkipFile;
181
+ /** 读文件头做内容嗅探;失败返回 null(不挡后续 parse) */
182
+ private readFileHead;
183
+ /** 尽量拿真实 identity;失败返回 undefined(退回 path: 降级键) */
184
+ private identityKeyFor;
105
185
  /**
106
- * 全量/增量索引某源(Phase A)
186
+ * 归属:`parse_file` / 读文件只读源 root 内的?
187
+ * ?**realpath** 比较,防符号链接逃出 root?
188
+ *
189
+ * @param source - 知识?
190
+ * @param path - 请求(本地绝对路径,
191
+ */
192
+ private sourceOwnsPath;
193
+ /**
194
+ * 全量/增量索引某源(Phase A?
195
+ *
196
+ * @param opts.full - 显式重建(supersede 成任务);poll/增量不打?
197
+ * @param opts.incremental - 增量(poll):?active 任务则跳过本?
107
198
  */
108
199
  ingestSource(sourceId: string, opts?: {
109
200
  full?: boolean;
201
+ incremental?: boolean;
202
+ fromQueue?: boolean;
110
203
  }): Promise<void>;
204
+ /**
205
+ * 入队层廉价变更检测:已完整索引且 size/mtime 未变则跳过 parse_file。
206
+ * 仅对 `status='indexed' && chunk_count>0 && content_hash` 生效——
207
+ * indexing/error/半成品必须重做,避免把未完成当成新鲜。
208
+ */
209
+ private isUnchangedIndexed;
111
210
  /**
112
211
  * URL / connector 源:discover + fetch + 索引
113
212
  */
114
213
  private ingestRemoteSource;
115
214
  /**
116
- * 解析源凭证;配置了 authRef 时 **fail-closed**(解析失败则本轮不抓,不降级匿名)。
215
+ * 解析源凭证;配置?authRef ?**fail-closed**(解析失败则不抓,不降级匿名?
117
216
  *
118
- * @returns 凭证或 null(仅当源未配置 authRef)
119
- * @throws authRef 配置了但无法解析时抛错,由调用方中止本轮
217
+ * @returns ?null(仅当源?authRef?
218
+ * @throws authRef 配置了但无法解析时抛错,由调用方
120
219
  */
121
220
  private resolveCred;
122
221
  /**
123
- * 将 VirtualDocument 写入索引(外源 / 规范化后文本)
222
+ * ?VirtualDocument 写入索引?/ 规范化后文本?
124
223
  *
125
- * @returns 是否成功写入
224
+ * @returns 成功写入
126
225
  */
127
- indexVirtualDoc(sourceId: string, doc: VirtualDocument): boolean;
226
+ indexVirtualDoc(sourceId: string, doc: VirtualDocument, authRef?: string | null): Promise<boolean>;
128
227
  /**
129
- * 单文件急索(P0 快车道;同步完成 parse,不 await 全库)
228
+ * 单文(P0 道;完成 parse,不 await 全库?
130
229
  */
131
230
  ingestFileNow(sourceId: string, filePath: string): Promise<boolean>;
231
+ /**
232
+ * 重做指定文件:强制重?解析 ?分块 ?向量
233
+ *
234
+ * - 必须归属(防越权读盘);外源逻辑 fetch_doc 重取
235
+ * - 失效 contentHash(忽?isFresh),?chunks 保留?upsert 成功
236
+ * - **于中 resume**:显式重?= 意图要跑活,不允许静默丢?
237
+ * - 等非文件:直接清,不入队
238
+ * - 完成后走既有 embed 链路
239
+ *
240
+ * @param sourceId - ?id
241
+ * @param paths - 文件列表
242
+ * @returns queued=新入队;alreadyActive=已在队列/;cleanedNonFiles=清掉的目录脏行;rejected=越权/不属
243
+ */
244
+ reprocessFiles(sourceId: string, paths: string[]): Promise<{
245
+ queued: number;
246
+ alreadyActive: number;
247
+ cleanedNonFiles: number;
248
+ resumed: boolean;
249
+ rejected: number;
250
+ }>;
251
+ /** 上是否已?queued/running ?parse/fetch 任务 */
252
+ private hasActiveParseJob;
253
+ /**
254
+ * 级任?文件UI 跟踪重做完成?
255
+ *
256
+ * @param sourceId - ?id
257
+ * @param paths - 列表
258
+ */
259
+ jobStateForPaths(sourceId: string, paths: string[]): Array<{
260
+ path: string;
261
+ jobsActive: number;
262
+ fileStatus: string | null;
263
+ chunkCount: number;
264
+ error: string | null;
265
+ exists: boolean;
266
+ }>;
267
+ /**
268
+ * 按筛选批量重做(当前列表?status/ext/q 条件?
269
+ *
270
+ * @param sourceId - ?id
271
+ * @param opts - 与文件列表筛选一?
272
+ * @returns 入队条数
273
+ */
274
+ reprocessByFilter(sourceId: string, opts?: {
275
+ status?: 'indexed' | 'skipped' | 'error' | 'all';
276
+ ext?: string;
277
+ q?: string;
278
+ }): Promise<{
279
+ queued: number;
280
+ alreadyActive: number;
281
+ cleanedNonFiles: number;
282
+ resumed: boolean;
283
+ rejected: number;
284
+ }>;
132
285
  /**
133
286
  * 处理 fs 变更(debounce 后入队)
134
287
  */
135
288
  handleFsChange(sourceId: string, filePath: string, event: 'change' | 'rename'): void;
136
289
  /**
137
- * 启动目录 watch(幂等)
290
+ * watch(幂等)
138
291
  */
139
292
  startWatch(sourceId: string): void;
140
293
  stopWatch(sourceId: string): void;
141
294
  /**
142
- * 等待队列排空(测试/管理)
295
+ * 等待队列排空(测?管理?
143
296
  */
144
297
  idle(timeoutMs?: number): Promise<void>;
145
298
  dispose(): void;
146
299
  /**
147
- * 启动 poll 调度(幂等);仅处理 sync.strategy=poll 且 enabled 的源
300
+ * 手动索引任务(业务完整:停收 + 清队 + 打断在跑?
301
+ *
302
+ * @param opts.sourceId - 仅中源;省略则全部源
303
+ * @returns 取消
304
+ */
305
+ abortJobs(opts?: {
306
+ sourceId?: string;
307
+ }): {
308
+ cancelledQueued: number;
309
+ abortedRunning: number;
310
+ runningJobs: number;
311
+ };
312
+ /**
313
+ * /继续索引(相轻)
314
+ *
315
+ * - 清除该源?
316
+ * - ?cancelled 任务放回 queued
317
+ * - 若仍缺向量则排队 embed
318
+ * - 不重?walk 全量;需要全 reindex
319
+ *
320
+ * @param opts.sourceId - 仅恢源;省略则全部源
321
+ * @returns
322
+ */
323
+ resumeJobs(opts?: {
324
+ sourceId?: string;
325
+ }): {
326
+ restoredCancelled: number;
327
+ embedQueued: number;
328
+ };
329
+ /** 的未(resume / supersede 后用;清 DB 态) */
330
+ private beginAbortEpoch;
331
+ /**
332
+ * 重建前作废本源未完成任务(supersede?
333
+ *
334
+ * - queued ?cancelled(superseded_by_reindex?
335
+ * - running 打断(worker/出)
336
+ * - 随后 beginAbortEpoch,本建可继续?
337
+ */
338
+ private supersedeSourceWork;
339
+ private ensureParseCoverage;
340
+ /** watch 突发/目录事件后:尽快做一次缺口补扫(默认 3s,不等 60s 节流) */
341
+ private requestGapScan;
342
+ /**
343
+ * 磁盘↔索齐:删掉已不存在的文件(?chunks/embeddings?
344
+ *
345
+ * 源用?discover walk ?keep 集(;失回并?stat?
346
+ * embed 前调避免给已移出文件的残 chunks 上向?
347
+ *
348
+ * @param sourceId - 知识?id
349
+ * @returns 删除的文件数
350
+ */
351
+ pruneMissingOnDisk(sourceId: string): Promise<number>;
352
+ /** 逐文?stat(file ?/ walk 失败);限流 + 单文件 removeFile(勿对每个 gone 做 removePathTree LIKE) */
353
+ private pruneByStatScan;
354
+ private abortSignalFor;
355
+ private isAborted;
356
+ /**
357
+ * /继续 控制数(互斥按钮
358
+ *
359
+ * @param sourceId - ?id
360
+ * @returns 队列计数 + 已中?+ 仍缺向量
361
+ */
362
+ jobControlState(sourceId: string): KnowledgeJobControlState;
363
+ /**
364
+ * 任务对账(看门狗?不依赖单?ensureEmbedJob
365
+ *
366
+ * 解决「续去重吞掉 / 进程 / partial」等静默停摆?
367
+ * 1. 回收超时 running(崩?孤儿?
368
+ * 2. 有缺向量且无 active embed ?排队
369
+ * 3. 无队缺口 ??coverage/status
370
+ */
371
+ startReconciler(intervalMs?: number): void;
372
+ stopReconciler(): void;
373
+ /** 对账入口(也工调 */
374
+ reconcileJobs(): Promise<void>;
375
+ /** ?job 清理(done/failed/cancelled 超过保留期) */
376
+ private cleanupTerminalJobs;
377
+ /** running 超时(无 heartbeat)→ 收回 queued次失败的直接?failed,防毒文 */
378
+ private reclaimStaleRunningJobs;
379
+ private countActiveJobs;
380
+ private hasRunningKind;
381
+ /**
382
+ * poll 调度(幂等);仅处理 sync.strategy=poll ?enabled 的源
148
383
  */
149
384
  startPolling(): void;
150
385
  stopPolling(): void;
151
386
  /**
152
- * 扫描到期 poll 源并入队刷新(内容未变由 content-hash / 304 短路)
387
+ * 到期 poll 源并入队刷新(内容未变由 content-hash / 304 ?
153
388
  *
154
- * @returns 本轮实际触发的 sourceId 列表
389
+ * @returns 实际触发?sourceId 列表
155
390
  */
156
391
  pollDueSources(): Promise<string[]>;
157
392
  private hasQueuedJobs;
393
+ /**
394
+ * 入队(同步调度用;失败不抛——DB 已关/中止时静默丢弃)。
395
+ * 需要计数的路径请 `await enqueueStrict`。
396
+ */
158
397
  private enqueue;
398
+ /** 入队并绑定真实 file_id(JobQueue 去重 / purge 清 job / abort 共享 File 守卫)。
399
+ *
400
+ * 共享 File:同 file_id 已有 parse 在途时只补 Membership,禁止双 parse。
401
+ */
402
+ private enqueueStrict;
403
+ /** parse/fetch 成功后回填 job.file_id(新建 File 的首条 job 入队时尚无 id) */
404
+ private bindJobFileId;
405
+ /** job 绑定 file_id:membership → identity;本地文件可 identify */
406
+ private resolveJobFileId;
159
407
  private kick;
160
408
  private drain;
409
+ /** 文档抽取单独限流:多份 PDF/Office 同时抽会打爆内存与 Engine */
410
+ private isDocParseJob;
411
+ private acceptParseSlot;
412
+ private countQueuedKinds;
161
413
  private claimJob;
162
414
  private runJob;
163
415
  /**
164
- * 外源单文档 job:按逻辑键 fetch + 索引
416
+ * 带截止时间的消执行:超时?abort signal,工作侧必须在副作用?
417
+ * 再用 Promise.race 包不消的 Promise会在超时后留下僵尸写?
418
+ */
419
+ private withTimeout;
420
+ /**
421
+ * 外源单文?job:按逻辑?fetch + 索引
165
422
  */
166
423
  private fetchDocJob;
167
424
  private parseOne;
168
425
  /**
169
- * PDF/Office:DocumentPort 抽取为 Markdown 后走 markdownAdapter 切块。
170
- * 新鲜度以原件字节 hash 为准。
426
+ * PDF/Office:DocumentPort 抽取?Markdown 后走 markdownAdapter 切块?
427
+ * 新鲜度以原件字节 hash 为准?
171
428
  *
172
- * @param sourceId - 知识源 id
173
- * @param filePath - 本地文件路径
174
- * @returns 是否成功入索引
429
+ * @param sourceId - 知识?id
430
+ * @param filePath - 文件
431
+ * @returns 成功入索?
175
432
  */
176
433
  private parseDocumentFile;
434
+ /** worker 抽取 + 流式 hash + 切块;注入 port 时(测试)由调用侧流式 hash */
435
+ private extractDocument;
436
+ /** 纯文本切块 + ftsToks(虚拟文档 / port 回退共用) */
437
+ private chunkTextInWorker;
177
438
  /**
178
- * Phase B:嵌入待处理 chunk(批 + 限速;失败不丢,下次再补)
439
+ * Phase B:嵌入待处理 chunk
440
+ *
441
+ * ?job 内按 embedConcurrency ****多批调用 API?
442
+ * 失败/超时?re-enqueue,避?job ?done ?coverage 永久卡住?
179
443
  */
180
444
  private embedPending;
445
+ private recordEmbedSecretHit;
446
+ /** 仍有缺向量则排队下一?embed_source(须在当?job ?done 之后调用?*/
447
+ private ensureEmbedJob;
448
+ private heartbeatJob;
449
+ /** parse 完成不立刻刷 coverage(O(N) 写放大);标脏,drain/reconcile 时合并刷?*/
450
+ private coverageDirty;
451
+ private markCoverageDirty;
452
+ private flushCoverageDirty;
453
+ private lastCoverageRefreshAt;
181
454
  private refreshCoverage;
182
455
  private emitProgress;
183
- /** 磁盘水位告警(仅告警,不丢任务) */
456
+ /** 磁盘水位(仅,不务) */
184
457
  private warnDiskWatermark;
185
458
  }