@octopi-agent/engine 0.56.0 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/context.js +1 -1
- package/dist/config-schema/engine.d.ts +43 -1
- package/dist/config-schema/substrate.d.ts +42 -0
- package/dist/config-schema/substrate.js +58 -0
- package/dist/config.d.ts +39 -1
- package/dist/harness/agent/builder.d.ts +14 -0
- package/dist/harness/agent/builder.js +14 -2
- package/dist/harness/agent/config-bridge.js +3 -0
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
- package/dist/harness/capabilities/document/factory.d.ts +33 -0
- package/dist/harness/capabilities/document/factory.js +37 -0
- package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
- package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
- package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
- package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
- package/dist/harness/capabilities/document/worker-port.js +78 -0
- package/dist/harness/context/capabilities/index.d.ts +2 -2
- package/dist/harness/context/capabilities/index.js +2 -2
- package/dist/harness/context/constitution/default-agents.md +1 -0
- package/dist/harness/context/index.d.ts +2 -2
- package/dist/harness/context/index.js +1 -1
- package/dist/harness/context/layer-types.d.ts +26 -2
- package/dist/harness/context/layer-types.js +59 -11
- package/dist/harness/context/layers.d.ts +3 -0
- package/dist/harness/context/layers.js +38 -12
- package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
- package/dist/harness/context/system-prompt-assembler.js +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +115 -5
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
- package/dist/harness/index.d.ts +5 -3
- package/dist/harness/index.js +3 -2
- package/dist/harness/knowledge/adapters.d.ts +8 -5
- package/dist/harness/knowledge/adapters.js +53 -13
- package/dist/harness/knowledge/catalog-types.d.ts +4 -0
- package/dist/harness/knowledge/client.d.ts +197 -0
- package/dist/harness/knowledge/client.js +237 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
- package/dist/harness/knowledge/connectors.d.ts +3 -3
- package/dist/harness/knowledge/connectors.js +14 -8
- package/dist/harness/knowledge/db.d.ts +42 -3
- package/dist/harness/knowledge/db.js +466 -83
- package/dist/harness/knowledge/describe.d.ts +2 -0
- package/dist/harness/knowledge/describe.js +5 -1
- package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
- package/dist/harness/knowledge/document-extract-worker.js +68 -0
- package/dist/harness/knowledge/embed-runner.d.ts +53 -0
- package/dist/harness/knowledge/embed-runner.js +171 -0
- package/dist/harness/knowledge/engine-thread.d.ts +1 -0
- package/dist/harness/knowledge/engine-thread.js +302 -0
- package/dist/harness/knowledge/extract-document.d.ts +33 -0
- package/dist/harness/knowledge/extract-document.js +86 -0
- package/dist/harness/knowledge/fetchers.d.ts +14 -3
- package/dist/harness/knowledge/fetchers.js +52 -16
- package/dist/harness/knowledge/file-identity.d.ts +53 -0
- package/dist/harness/knowledge/file-identity.js +172 -0
- package/dist/harness/knowledge/file-index-store.d.ts +50 -0
- package/dist/harness/knowledge/file-index-store.js +132 -0
- package/dist/harness/knowledge/file-limits.d.ts +86 -0
- package/dist/harness/knowledge/file-limits.js +175 -0
- package/dist/harness/knowledge/fts.d.ts +60 -0
- package/dist/harness/knowledge/fts.js +218 -0
- package/dist/harness/knowledge/grounding.d.ts +3 -2
- package/dist/harness/knowledge/http-app.d.ts +50 -0
- package/dist/harness/knowledge/http-app.js +652 -0
- package/dist/harness/knowledge/http-bridge.d.ts +49 -0
- package/dist/harness/knowledge/http-bridge.js +17 -0
- package/dist/harness/knowledge/index-store.d.ts +166 -63
- package/dist/harness/knowledge/index-store.js +1025 -282
- package/dist/harness/knowledge/index.d.ts +47 -5
- package/dist/harness/knowledge/index.js +24 -2
- package/dist/harness/knowledge/ingest.d.ts +322 -49
- package/dist/harness/knowledge/ingest.js +1383 -267
- package/dist/harness/knowledge/job-control-state.d.ts +29 -0
- package/dist/harness/knowledge/job-control-state.js +44 -0
- package/dist/harness/knowledge/job-control.d.ts +35 -0
- package/dist/harness/knowledge/job-control.js +86 -0
- package/dist/harness/knowledge/job-queue.d.ts +62 -0
- package/dist/harness/knowledge/job-queue.js +173 -0
- package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
- package/dist/harness/knowledge/local-stack.d.ts +27 -0
- package/dist/harness/knowledge/local-stack.js +63 -0
- package/dist/harness/knowledge/membership-store.d.ts +65 -0
- package/dist/harness/knowledge/membership-store.js +198 -0
- package/dist/harness/knowledge/network-guard.d.ts +11 -2
- package/dist/harness/knowledge/network-guard.js +130 -85
- package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
- package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
- package/dist/harness/knowledge/query-service.d.ts +272 -0
- package/dist/harness/knowledge/query-service.js +326 -0
- package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
- package/dist/harness/knowledge/query-worker-client.js +207 -0
- package/dist/harness/knowledge/query-worker.d.ts +1 -0
- package/dist/harness/knowledge/query-worker.js +186 -0
- package/dist/harness/knowledge/read-http.d.ts +35 -0
- package/dist/harness/knowledge/read-http.js +373 -0
- package/dist/harness/knowledge/retriever.d.ts +30 -5
- package/dist/harness/knowledge/retriever.js +98 -35
- package/dist/harness/knowledge/secret-scan.d.ts +13 -2
- package/dist/harness/knowledge/secret-scan.js +25 -2
- package/dist/harness/knowledge/serve.d.ts +48 -0
- package/dist/harness/knowledge/serve.js +336 -0
- package/dist/harness/knowledge/source-store.d.ts +83 -16
- package/dist/harness/knowledge/source-store.js +205 -78
- package/dist/harness/knowledge/start-service-process.d.ts +49 -0
- package/dist/harness/knowledge/start-service-process.js +158 -0
- package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
- package/dist/harness/knowledge/text-parse-worker.js +54 -0
- package/dist/harness/knowledge/topics.d.ts +13 -0
- package/dist/harness/knowledge/topics.js +48 -0
- package/dist/harness/knowledge/types.d.ts +1 -1
- package/dist/harness/knowledge/types.js +1 -1
- package/dist/harness/knowledge/vector-ann.d.ts +42 -0
- package/dist/harness/knowledge/vector-ann.js +68 -0
- package/dist/harness/knowledge/writer-lock.d.ts +13 -0
- package/dist/harness/knowledge/writer-lock.js +71 -0
- package/dist/harness/knowledge/writer-service.d.ts +194 -0
- package/dist/harness/knowledge/writer-service.js +355 -0
- package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
- package/dist/harness/knowledge/writer-worker-client.js +381 -0
- package/dist/harness/knowledge/writer-worker.d.ts +1 -0
- package/dist/harness/knowledge/writer-worker.js +189 -0
- package/dist/harness/memory/index.d.ts +3 -2
- package/dist/harness/memory/index.js +3 -1
- package/dist/harness/memory/retrieval-rank.d.ts +51 -0
- package/dist/harness/memory/retrieval-rank.js +60 -0
- package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
- package/dist/harness/memory/sqlite/embedding.js +109 -16
- package/dist/harness/memory/sqlite/index.d.ts +2 -2
- package/dist/harness/memory/sqlite/index.js +1 -1
- package/dist/harness/memory/sqlite/keyword-search.d.ts +41 -2
- package/dist/harness/memory/sqlite/keyword-search.js +104 -17
- package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
- package/dist/harness/memory/sqlite/memory-store.js +89 -39
- package/dist/harness/memory/store.js +39 -21
- package/dist/harness/memory/types.d.ts +7 -0
- package/dist/harness/run/runner.d.ts +1 -0
- package/dist/harness/run/runner.js +4 -2
- package/dist/harness/session/attachments/service.d.ts +1 -1
- package/dist/harness/session/attachments/service.js +1 -1
- package/dist/harness/session/tasks/index.d.ts +1 -1
- package/dist/harness/session/tasks/index.js +1 -1
- package/dist/harness/session/tasks/render.d.ts +10 -0
- package/dist/harness/session/tasks/render.js +27 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2 -1
- package/dist/worker-path.d.ts +8 -0
- package/dist/worker-path.js +15 -0
- package/package.json +2 -2
|
@@ -6,23 +6,62 @@
|
|
|
6
6
|
*/
|
|
7
7
|
export { KnowledgeDatabase } from './db.js';
|
|
8
8
|
export type { KnowledgeDatabaseOptions } from './db.js';
|
|
9
|
+
export { identifyLocalFile, identifyUrl, identifyConnector, normalizeFsPath, normalizePathLexical, urlIdentityKey, connectorIdentityKey, logicalPathFrom, isPathPrefix, canonicalizeUrl, } from './file-identity.js';
|
|
10
|
+
export type { FileIdentity } from './file-identity.js';
|
|
11
|
+
export { MembershipStore } from './membership-store.js';
|
|
12
|
+
export type { MembershipRow, FileRow, ReconcileResult } from './membership-store.js';
|
|
13
|
+
export { FileIndexStore } from './file-index-store.js';
|
|
14
|
+
export type { ChunkDraft, UpsertFileResult } from './file-index-store.js';
|
|
9
15
|
export { KnowledgeSourceStore } from './source-store.js';
|
|
10
16
|
export { KnowledgeIndexStore, hashContent } from './index-store.js';
|
|
11
17
|
export type { IndexedFileRecord, ChunkHit } from './index-store.js';
|
|
18
|
+
export { KnowledgeFts, buildFtsTokens, buildFtsQuery } from './fts.js';
|
|
19
|
+
export { vectorBucket, queryBuckets, scoreAnnCandidates, VECTOR_BUCKETS, } from './vector-ann.js';
|
|
12
20
|
export { FormatAdapterRegistry, textAdapter, markdownAdapter, htmlAdapter, codeAdapter, chunkCodeBySymbols, IGNORED_DIRS, BINARY_EXTENSIONS, } from './adapters.js';
|
|
13
21
|
export type { FormatAdapter, KnowledgeChunkDraft } from './adapters.js';
|
|
14
22
|
export { KnowledgeIngest } from './ingest.js';
|
|
15
|
-
export type { KnowledgeIngestOptions, IngestProgressEvent, IngestJobKind } from './ingest.js';
|
|
23
|
+
export type { KnowledgeIngestOptions, IngestProgressEvent, IngestJobKind, } from './ingest.js';
|
|
24
|
+
export { KnowledgeJobControl } from './job-control.js';
|
|
25
|
+
export { JobQueue } from './job-queue.js';
|
|
26
|
+
export type { IngestJobKind as JobKind, JobRow } from './job-queue.js';
|
|
27
|
+
export { EmbedRunner } from './embed-runner.js';
|
|
28
|
+
export type { EmbedSecretPolicy } from './embed-runner.js';
|
|
29
|
+
export { DEFAULT_KNOWLEDGE_FILE_LIMITS, classifyFileKind, decideBySize, formatMaxBytes, isRetryableSkipReason, parseTimeoutForSize, resolveKnowledgeFileLimits, } from './file-limits.js';
|
|
30
|
+
export type { KnowledgeFileKind, KnowledgeFileLimits, KnowledgeFileLimitsInput, OversizePolicy, } from './file-limits.js';
|
|
16
31
|
export { LocalFsFetcher, UrlFetcher, } from './fetchers.js';
|
|
17
|
-
export type { SourceFetcher, DiscoveredDocRef, VirtualDocument, } from './fetchers.js';
|
|
32
|
+
export type { SourceFetcher, DiscoveredDocRef, DiscoverResult, VirtualDocument, } from './fetchers.js';
|
|
18
33
|
export { ConnectorRegistry, RestConnector } from './connectors.js';
|
|
19
34
|
export type { KnowledgeConnector, ConnectorContext, RestConnectorConfig } from './connectors.js';
|
|
20
35
|
export { ConnectorFetcher } from './connector-fetcher.js';
|
|
21
|
-
export { assertUrlAllowed, guardedFetch, isRestrictedIp } from './network-guard.js';
|
|
36
|
+
export { assertUrlAllowed, authHeadersForUrl, guardedFetch, isRestrictedIp } from './network-guard.js';
|
|
22
37
|
export type { NetworkGuardOptions, FetchResult } from './network-guard.js';
|
|
23
38
|
export { htmlToStructuredText, looksLikeHtml } from './html.js';
|
|
24
39
|
export { KnowledgeRetriever } from './retriever.js';
|
|
25
|
-
export
|
|
40
|
+
export { LocalKnowledgeQueryService, createKnowledgeQueryService, } from './query-service.js';
|
|
41
|
+
export type { KnowledgeQueryService, SearchQuery, AutoGroundQuery, ListSourcesQuery, QueryIdentity, PrincipalStats, ReadChunkResult, SourceStatsBundle, SourceDetail, ListJobsQuery, ReadySnapshot, QueryWorkerRole, } from './query-service.js';
|
|
42
|
+
export { readJobControlState } from './job-control-state.js';
|
|
43
|
+
export type { KnowledgeJobControlState } from './job-control-state.js';
|
|
44
|
+
export { handleKnowledgeReadHttp, isPureReadRoute, authenticateRead, } from './read-http.js';
|
|
45
|
+
export type { ReadHttpDeps } from './read-http.js';
|
|
46
|
+
export { KnowledgeHttpApp, createKnowledgeHttpApp } from './http-app.js';
|
|
47
|
+
export type { KnowledgeServiceToken, KnowledgeServiceOptions, AuthContext } from './http-app.js';
|
|
48
|
+
export { LocalKnowledgeWriteService, } from './writer-service.js';
|
|
49
|
+
export type { KnowledgeWriteService, WriteIdentity, WriteAbortStats, WriteResumeStats, WriteRegisterResult, WriteDescribeResult, WriteReprocessResult, } from './writer-service.js';
|
|
50
|
+
export { WorkerWriteService } from './writer-worker-client.js';
|
|
51
|
+
export type { WorkerWriteStartOptions } from './writer-worker-client.js';
|
|
52
|
+
export { createLocalKnowledgeStack } from './local-stack.js';
|
|
53
|
+
export type { LocalKnowledgeStack } from './local-stack.js';
|
|
54
|
+
export { startKnowledgeService } from './serve.js';
|
|
55
|
+
export type { KnowledgeServeOptions, KnowledgeServeHandle } from './serve.js';
|
|
56
|
+
export { startKnowledgeServiceProcess } from './start-service-process.js';
|
|
57
|
+
export type { KnowledgeServiceProcessOptions, KnowledgeServiceProcessHandle, } from './start-service-process.js';
|
|
58
|
+
export { acquireKnowledgeWriterLock } from './writer-lock.js';
|
|
59
|
+
export type { WriterLockHolder } from './writer-lock.js';
|
|
60
|
+
export { parseTextInWorker } from './parse-text-in-worker.js';
|
|
61
|
+
export type { TextParseResult, TextParseWorkerOptions } from './parse-text-in-worker.js';
|
|
62
|
+
export { KnowledgeClient, ClientKnowledgeGrounding } from './client.js';
|
|
63
|
+
export type { KnowledgeClientOptions } from './client.js';
|
|
64
|
+
export type { HybridSearchOptions, HybridSearchResult, AutoGroundDecision, AutoGroundPort, GroundingMode, KnowledgeRecallMode, KnowledgeRetrieverOptions, } from './retriever.js';
|
|
26
65
|
export { GroundingAssembler, formatKnowledgeGroundingMessage, resolveGroundingQuery, wrapUntrustedKnowledgeBlock, isKnowledgeGroundingMessage, stripKnowledgeGrounding, KNOWLEDGE_GROUNDING_SOURCE, } from './grounding.js';
|
|
27
66
|
export type { GroundingPack, GroundingAssemblerOptions } from './grounding.js';
|
|
28
67
|
export { KnowledgeHitLog } from './hit-log.js';
|
|
@@ -31,8 +70,11 @@ export { KnowledgePurger } from './purge.js';
|
|
|
31
70
|
export type { PurgeResult } from './purge.js';
|
|
32
71
|
export { resolveKnowledgePaths } from './paths.js';
|
|
33
72
|
export type { KnowledgePaths } from './paths.js';
|
|
73
|
+
export { deriveTopicsFromPaths } from './topics.js';
|
|
74
|
+
export type { KnowledgeCatalogItem, KnowledgeCatalogProvider } from './catalog-types.js';
|
|
34
75
|
export { generateKnowledgeDescription, heuristicDescription, } from './describe.js';
|
|
35
76
|
export type { KnowledgeDescribePort, KnowledgeDescribeOptions, KnowledgeDescribeResult, } from './describe.js';
|
|
36
|
-
export { scanSecretShapes } from './secret-scan.js';
|
|
77
|
+
export { scanSecretShapes, redactSecretShapes } from './secret-scan.js';
|
|
78
|
+
export type { RedactResult } from './secret-scan.js';
|
|
37
79
|
export type { KnowledgeSource, KnowledgeSourceInput, KnowledgeSourcePatch, KnowledgeSourceKind, KnowledgeSourceStatus, KnowledgeSourceSync, KnowledgeSourceError, KnowledgeSourceNetwork, KnowledgeSourceDiscover, KnowledgeScopeRef, KnowledgeScopeLevel, KnowledgeSourceId, KnowledgeChunkId, KnowledgeVisibilityTargetType, KnowledgeVisibilityOp, KnowledgeSessionVisibilityItem, KnowledgeSessionVisibilityInput, Branded, } from './types.js';
|
|
38
80
|
export { asSourceId, asChunkId } from './types.js';
|
|
@@ -5,20 +5,42 @@
|
|
|
5
5
|
* 规格:arch/knowledge-layer.md
|
|
6
6
|
*/
|
|
7
7
|
export { KnowledgeDatabase } from './db.js';
|
|
8
|
+
export { identifyLocalFile, identifyUrl, identifyConnector, normalizeFsPath, normalizePathLexical, urlIdentityKey, connectorIdentityKey, logicalPathFrom, isPathPrefix, canonicalizeUrl, } from './file-identity.js';
|
|
9
|
+
export { MembershipStore } from './membership-store.js';
|
|
10
|
+
export { FileIndexStore } from './file-index-store.js';
|
|
8
11
|
export { KnowledgeSourceStore } from './source-store.js';
|
|
9
12
|
export { KnowledgeIndexStore, hashContent } from './index-store.js';
|
|
13
|
+
export { KnowledgeFts, buildFtsTokens, buildFtsQuery } from './fts.js';
|
|
14
|
+
export { vectorBucket, queryBuckets, scoreAnnCandidates, VECTOR_BUCKETS, } from './vector-ann.js';
|
|
10
15
|
export { FormatAdapterRegistry, textAdapter, markdownAdapter, htmlAdapter, codeAdapter, chunkCodeBySymbols, IGNORED_DIRS, BINARY_EXTENSIONS, } from './adapters.js';
|
|
11
16
|
export { KnowledgeIngest } from './ingest.js';
|
|
17
|
+
export { KnowledgeJobControl } from './job-control.js';
|
|
18
|
+
export { JobQueue } from './job-queue.js';
|
|
19
|
+
export { EmbedRunner } from './embed-runner.js';
|
|
20
|
+
export { DEFAULT_KNOWLEDGE_FILE_LIMITS, classifyFileKind, decideBySize, formatMaxBytes, isRetryableSkipReason, parseTimeoutForSize, resolveKnowledgeFileLimits, } from './file-limits.js';
|
|
12
21
|
export { LocalFsFetcher, UrlFetcher, } from './fetchers.js';
|
|
13
22
|
export { ConnectorRegistry, RestConnector } from './connectors.js';
|
|
14
23
|
export { ConnectorFetcher } from './connector-fetcher.js';
|
|
15
|
-
export { assertUrlAllowed, guardedFetch, isRestrictedIp } from './network-guard.js';
|
|
24
|
+
export { assertUrlAllowed, authHeadersForUrl, guardedFetch, isRestrictedIp } from './network-guard.js';
|
|
16
25
|
export { htmlToStructuredText, looksLikeHtml } from './html.js';
|
|
17
26
|
export { KnowledgeRetriever } from './retriever.js';
|
|
27
|
+
export { LocalKnowledgeQueryService, createKnowledgeQueryService, } from './query-service.js';
|
|
28
|
+
export { readJobControlState } from './job-control-state.js';
|
|
29
|
+
export { handleKnowledgeReadHttp, isPureReadRoute, authenticateRead, } from './read-http.js';
|
|
30
|
+
export { KnowledgeHttpApp, createKnowledgeHttpApp } from './http-app.js';
|
|
31
|
+
export { LocalKnowledgeWriteService, } from './writer-service.js';
|
|
32
|
+
export { WorkerWriteService } from './writer-worker-client.js';
|
|
33
|
+
export { createLocalKnowledgeStack } from './local-stack.js';
|
|
34
|
+
export { startKnowledgeService } from './serve.js';
|
|
35
|
+
export { startKnowledgeServiceProcess } from './start-service-process.js';
|
|
36
|
+
export { acquireKnowledgeWriterLock } from './writer-lock.js';
|
|
37
|
+
export { parseTextInWorker } from './parse-text-in-worker.js';
|
|
38
|
+
export { KnowledgeClient, ClientKnowledgeGrounding } from './client.js';
|
|
18
39
|
export { GroundingAssembler, formatKnowledgeGroundingMessage, resolveGroundingQuery, wrapUntrustedKnowledgeBlock, isKnowledgeGroundingMessage, stripKnowledgeGrounding, KNOWLEDGE_GROUNDING_SOURCE, } from './grounding.js';
|
|
19
40
|
export { KnowledgeHitLog } from './hit-log.js';
|
|
20
41
|
export { KnowledgePurger } from './purge.js';
|
|
21
42
|
export { resolveKnowledgePaths } from './paths.js';
|
|
43
|
+
export { deriveTopicsFromPaths } from './topics.js';
|
|
22
44
|
export { generateKnowledgeDescription, heuristicDescription, } from './describe.js';
|
|
23
|
-
export { scanSecretShapes } from './secret-scan.js';
|
|
45
|
+
export { scanSecretShapes, redactSecretShapes } from './secret-scan.js';
|
|
24
46
|
export { asSourceId, asChunkId } from './types.js';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* KnowledgeIngest
|
|
2
|
+
* KnowledgeIngest ?解析/切块/索引队列(P2 Phase A?
|
|
3
3
|
*
|
|
4
|
-
* 原则:ingest
|
|
4
|
+
* 原则:ingest 话并发,不作 turn 前置;队列背压不务;source 级锁?
|
|
5
5
|
*/
|
|
6
6
|
import { EventEmitter } from 'node:events';
|
|
7
7
|
import { FormatAdapterRegistry } from './adapters.js';
|
|
@@ -11,63 +11,105 @@ import { type SourceFetcher, type VirtualDocument } from './fetchers.js';
|
|
|
11
11
|
import { ConnectorRegistry } from './connectors.js';
|
|
12
12
|
import type { CredentialStore } from '../governance/credentials/store.js';
|
|
13
13
|
import type { EmbeddingProvider } from '../memory/sqlite/embedding.js';
|
|
14
|
-
import type { DocumentPort } from '../
|
|
15
|
-
|
|
14
|
+
import type { DocumentPort } from '../capabilities/document/types.js';
|
|
15
|
+
import { type KnowledgeFileLimitsInput } from './file-limits.js';
|
|
16
|
+
import { type EmbedSecretPolicy } from './embed-runner.js';
|
|
17
|
+
/**
|
|
18
|
+
* 流式 SHA-256(可中止):禁止在 Engine 事件循环 `readFile` 整文件再 hash。
|
|
19
|
+
*
|
|
20
|
+
* @param filePath - 本地文件
|
|
21
|
+
* @param signal - 中止信号
|
|
22
|
+
* @returns hex digest
|
|
23
|
+
*/
|
|
24
|
+
export declare function hashFileStreaming(filePath: string, signal?: AbortSignal): Promise<string>;
|
|
25
|
+
import { type IngestJobKind as JobKind } from './job-queue.js';
|
|
26
|
+
import { type KnowledgeJobControlState } from './job-control-state.js';
|
|
27
|
+
export type { EmbedSecretPolicy };
|
|
28
|
+
export type IngestJobKind = JobKind;
|
|
29
|
+
export type { KnowledgeJobControlState };
|
|
16
30
|
export interface IngestProgressEvent {
|
|
17
31
|
type: 'job' | 'source' | 'error' | 'embed';
|
|
18
32
|
sourceId: string;
|
|
19
33
|
path?: string;
|
|
20
34
|
status: string;
|
|
21
35
|
detail?: string;
|
|
36
|
+
/** SSE 过滤用:源归属(Writer 在 emit 时补齐,API 不得再 RPC 查) */
|
|
37
|
+
tenantId?: string;
|
|
38
|
+
registeredBy?: string;
|
|
39
|
+
visibility?: string;
|
|
22
40
|
}
|
|
23
41
|
export interface KnowledgeIngestOptions {
|
|
24
42
|
sourceStore: KnowledgeSourceStore;
|
|
25
43
|
indexStore?: KnowledgeIndexStore;
|
|
26
44
|
adapterRegistry?: FormatAdapterRegistry;
|
|
27
|
-
/** parse 并发(默认
|
|
45
|
+
/** parse 并发(默认 8) */
|
|
28
46
|
parseConcurrency?: number;
|
|
29
|
-
/**
|
|
47
|
+
/** 文档(PDF/Office)抽取并发(默认 2;大文件抢占 CPU/内存) */
|
|
48
|
+
documentParseConcurrency?: number;
|
|
49
|
+
/** 队列深度上限(超出只,不务) */
|
|
30
50
|
maxQueueDepth?: number;
|
|
31
|
-
/** fs watch debounce ms
|
|
51
|
+
/** fs watch debounce ms(默?2000?*/
|
|
32
52
|
debounceMs?: number;
|
|
33
|
-
/**
|
|
53
|
+
/** 单文件大小兜底上限(格式分级?fileLimits?*/
|
|
34
54
|
maxFileBytes?: number;
|
|
35
|
-
/**
|
|
55
|
+
/**
|
|
56
|
+
* 按格式分级的文件 + 部分抽取策略?
|
|
57
|
+
* 机器资源不足时下?maxBytes / hardMaxFileBytes?
|
|
58
|
+
*/
|
|
59
|
+
fileLimits?: KnowledgeFileLimitsInput;
|
|
60
|
+
/** 单文?parse 超时 ms(默?45000?*/
|
|
61
|
+
parseTimeoutMs?: number;
|
|
62
|
+
/** Phase B embedding(未配则?*/
|
|
36
63
|
embeddingProvider?: EmbeddingProvider | null;
|
|
37
|
-
/** embed
|
|
64
|
+
/** embed 批大小( 32?*/
|
|
38
65
|
embedBatch?: number;
|
|
39
|
-
/** embed
|
|
66
|
+
/** embed 小间?ms(限速, 0?*/
|
|
40
67
|
embedMinIntervalMs?: number;
|
|
41
|
-
/** embed
|
|
68
|
+
/** embed 并发(默?parseConcurrency?*/
|
|
42
69
|
embedConcurrency?: number;
|
|
43
|
-
/**
|
|
70
|
+
/** 磁盘水位(默?false;仅不丢任务?*/
|
|
44
71
|
diskWatermarkAlert?: boolean;
|
|
45
|
-
/**
|
|
72
|
+
/** 库(source.authRef 解析;可选) */
|
|
46
73
|
credentials?: CredentialStore | null;
|
|
47
|
-
/**
|
|
74
|
+
/** ?fetcher 覆盖(测?扩展?*/
|
|
48
75
|
fetchers?: Partial<Record<'local' | 'url' | 'connector', SourceFetcher>>;
|
|
49
|
-
/** connector
|
|
76
|
+
/** connector 注册?rest?*/
|
|
50
77
|
connectors?: ConnectorRegistry;
|
|
51
|
-
/** poll 调度 tick ms
|
|
78
|
+
/** poll 调度 tick ms(默?60s?*/
|
|
52
79
|
pollTickMs?: number;
|
|
53
|
-
/**
|
|
80
|
+
/** 单源?poll 间隔 ms(默?15min?*/
|
|
54
81
|
pollMinIntervalMs?: number;
|
|
55
|
-
/** 每轮 poll
|
|
82
|
+
/** 每轮 poll 理源数(成本上限,默?8?*/
|
|
56
83
|
maxPollPerTick?: number;
|
|
57
84
|
/**
|
|
58
|
-
* DocumentPort:启用后 PDF/Office 等走抽取
|
|
59
|
-
*
|
|
85
|
+
* DocumentPort:启用后 PDF/Office 等走抽取 ?Markdown 切块
|
|
86
|
+
* (缺?null = 维持 BINARY skip ?
|
|
60
87
|
*/
|
|
61
88
|
documentPort?: DocumentPort | null;
|
|
89
|
+
/**
|
|
90
|
+
* documents.* 配置 — 与 Gateway 同源;worker 抽取与 legacy/soffice 共用
|
|
91
|
+
*/
|
|
92
|
+
documentConfig?: import('../capabilities/document/factory.js').DocumentCapabilityConfig | null;
|
|
93
|
+
/**
|
|
94
|
+
* embedding 外发前敏感形态策略( redact?
|
|
95
|
+
* - allow:原发(内网/模型?
|
|
96
|
+
* - redact:命 [REDACTED:rule] ?embed
|
|
97
|
+
* - skip:命不写向量(关仍可搜)
|
|
98
|
+
*/
|
|
99
|
+
embedSecretPolicy?: EmbedSecretPolicy;
|
|
62
100
|
}
|
|
63
101
|
export declare class KnowledgeIngest extends EventEmitter {
|
|
64
102
|
private readonly sources;
|
|
65
103
|
private readonly index;
|
|
66
104
|
private readonly adapters;
|
|
67
105
|
private readonly parseConcurrency;
|
|
106
|
+
private readonly documentParseConcurrency;
|
|
68
107
|
private readonly maxQueueDepth;
|
|
69
108
|
private readonly debounceMs;
|
|
70
109
|
private readonly maxFileBytes;
|
|
110
|
+
private readonly fileLimits;
|
|
111
|
+
/** 单文?parse/fetch 超时(默?45s;大 Office 抽取会堵事件?*/
|
|
112
|
+
private readonly parseTimeoutMs;
|
|
71
113
|
private readonly embedding;
|
|
72
114
|
private readonly embedBatch;
|
|
73
115
|
private readonly embedMinIntervalMs;
|
|
@@ -81,105 +123,336 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
81
123
|
private readonly pollMinIntervalMs;
|
|
82
124
|
private readonly maxPollPerTick;
|
|
83
125
|
private readonly documentPort;
|
|
84
|
-
private
|
|
126
|
+
private readonly documentConfig;
|
|
127
|
+
private readonly embedSecretPolicy;
|
|
128
|
+
private readonly embedRunner;
|
|
85
129
|
private pollTimer;
|
|
86
|
-
private
|
|
130
|
+
private reconcileTimer;
|
|
131
|
+
/** 源级:DB 权威 + ?AbortSignal(跨重启?*/
|
|
132
|
+
private readonly jobControl;
|
|
133
|
+
private readonly jobQueue;
|
|
134
|
+
/** 源任务稳定(?queued/running)时回调 ?用于 auto-describe 等收?*/
|
|
135
|
+
onSourceSettled?: (sourceId: string) => void;
|
|
136
|
+
/** parse 缺口时间 */
|
|
137
|
+
private lastParseGapScanAt;
|
|
138
|
+
/** watch 突发后要求尽快缺口补扫(覆盖 60s 节流) */
|
|
139
|
+
private gapScanUrgent;
|
|
140
|
+
private gapScanTimer;
|
|
141
|
+
private runningParse;
|
|
142
|
+
private runningDocParse;
|
|
87
143
|
private runningEmbed;
|
|
88
144
|
private draining;
|
|
89
|
-
|
|
145
|
+
private kickAgain;
|
|
146
|
+
/** source 级锁:同 source 不并?walk/parse 冲突?*/
|
|
90
147
|
private sourceLocks;
|
|
91
|
-
/** walk 发现的文件数(Phase A coverage
|
|
148
|
+
/** walk 发现的文件数(Phase A coverage 分母?*/
|
|
92
149
|
private discoveredFiles;
|
|
93
150
|
private watchers;
|
|
94
151
|
private debounceTimers;
|
|
95
152
|
private disposed;
|
|
96
153
|
constructor(options: KnowledgeIngestOptions);
|
|
154
|
+
private lastCleanupAt;
|
|
155
|
+
/**
|
|
156
|
+
* 清掉「非?knowledge_files (目录曾 no_adapter?
|
|
157
|
+
*
|
|
158
|
+
* ?skipped/error 且无 chunk :有 chunk 的仍?pruneMissing/磁盘对齐?
|
|
159
|
+
* 60s 节流,避免每?reconcile ?stat?
|
|
160
|
+
*
|
|
161
|
+
* @param sourceId - 限定源;省略则全部源
|
|
162
|
+
* @param opts.force - 忽略节流
|
|
163
|
+
* @returns 删除行数
|
|
164
|
+
*/
|
|
165
|
+
cleanupNonFileIndexRows(sourceId?: string, opts?: {
|
|
166
|
+
force?: boolean;
|
|
167
|
+
}): number;
|
|
168
|
+
/**
|
|
169
|
+
* 无跨进程 Lease:本进程时把历史 running 收回 queued?
|
|
170
|
+
* 仅构造时调用行中同进程的 running 法的?
|
|
171
|
+
*/
|
|
172
|
+
private reclaimOrphanRunningJobs;
|
|
97
173
|
get indexStore(): KnowledgeIndexStore;
|
|
98
174
|
/**
|
|
99
|
-
*
|
|
175
|
+
* 过滤:文档扩展名交给 worker 抽取,不 BINARY ?
|
|
100
176
|
*
|
|
101
|
-
* @param p -
|
|
102
|
-
* @returns
|
|
177
|
+
* @param p - 文件
|
|
178
|
+
* @returns 跳过
|
|
103
179
|
*/
|
|
104
180
|
private shouldSkipFile;
|
|
181
|
+
/** 读文件头做内容嗅探;失败返回 null(不挡后续 parse) */
|
|
182
|
+
private readFileHead;
|
|
183
|
+
/** 尽量拿真实 identity;失败返回 undefined(退回 path: 降级键) */
|
|
184
|
+
private identityKeyFor;
|
|
105
185
|
/**
|
|
106
|
-
*
|
|
186
|
+
* 归属:`parse_file` / 读文件只读源 root 内的?
|
|
187
|
+
* ?**realpath** 比较,防符号链接逃出 root?
|
|
188
|
+
*
|
|
189
|
+
* @param source - 知识?
|
|
190
|
+
* @param path - 请求(本地绝对路径,
|
|
191
|
+
*/
|
|
192
|
+
private sourceOwnsPath;
|
|
193
|
+
/**
|
|
194
|
+
* 全量/增量索引某源(Phase A?
|
|
195
|
+
*
|
|
196
|
+
* @param opts.full - 显式重建(supersede 成任务);poll/增量不打?
|
|
197
|
+
* @param opts.incremental - 增量(poll):?active 任务则跳过本?
|
|
107
198
|
*/
|
|
108
199
|
ingestSource(sourceId: string, opts?: {
|
|
109
200
|
full?: boolean;
|
|
201
|
+
incremental?: boolean;
|
|
202
|
+
fromQueue?: boolean;
|
|
110
203
|
}): Promise<void>;
|
|
204
|
+
/**
|
|
205
|
+
* 入队层廉价变更检测:已完整索引且 size/mtime 未变则跳过 parse_file。
|
|
206
|
+
* 仅对 `status='indexed' && chunk_count>0 && content_hash` 生效——
|
|
207
|
+
* indexing/error/半成品必须重做,避免把未完成当成新鲜。
|
|
208
|
+
*/
|
|
209
|
+
private isUnchangedIndexed;
|
|
111
210
|
/**
|
|
112
211
|
* URL / connector 源:discover + fetch + 索引
|
|
113
212
|
*/
|
|
114
213
|
private ingestRemoteSource;
|
|
115
214
|
/**
|
|
116
|
-
*
|
|
215
|
+
* 解析源凭证;配置?authRef ?**fail-closed**(解析失败则不抓,不降级匿名?
|
|
117
216
|
*
|
|
118
|
-
* @returns
|
|
119
|
-
* @throws authRef
|
|
217
|
+
* @returns ?null(仅当源?authRef?
|
|
218
|
+
* @throws authRef 配置了但无法解析时抛错,由调用方
|
|
120
219
|
*/
|
|
121
220
|
private resolveCred;
|
|
122
221
|
/**
|
|
123
|
-
*
|
|
222
|
+
* ?VirtualDocument 写入索引?/ 规范化后文本?
|
|
124
223
|
*
|
|
125
|
-
* @returns
|
|
224
|
+
* @returns 成功写入
|
|
126
225
|
*/
|
|
127
|
-
indexVirtualDoc(sourceId: string, doc: VirtualDocument): boolean
|
|
226
|
+
indexVirtualDoc(sourceId: string, doc: VirtualDocument, authRef?: string | null): Promise<boolean>;
|
|
128
227
|
/**
|
|
129
|
-
*
|
|
228
|
+
* 单文(P0 道;完成 parse,不 await 全库?
|
|
130
229
|
*/
|
|
131
230
|
ingestFileNow(sourceId: string, filePath: string): Promise<boolean>;
|
|
231
|
+
/**
|
|
232
|
+
* 重做指定文件:强制重?解析 ?分块 ?向量
|
|
233
|
+
*
|
|
234
|
+
* - 必须归属(防越权读盘);外源逻辑 fetch_doc 重取
|
|
235
|
+
* - 失效 contentHash(忽?isFresh),?chunks 保留?upsert 成功
|
|
236
|
+
* - **于中 resume**:显式重?= 意图要跑活,不允许静默丢?
|
|
237
|
+
* - 等非文件:直接清,不入队
|
|
238
|
+
* - 完成后走既有 embed 链路
|
|
239
|
+
*
|
|
240
|
+
* @param sourceId - ?id
|
|
241
|
+
* @param paths - 文件列表
|
|
242
|
+
* @returns queued=新入队;alreadyActive=已在队列/;cleanedNonFiles=清掉的目录脏行;rejected=越权/不属
|
|
243
|
+
*/
|
|
244
|
+
reprocessFiles(sourceId: string, paths: string[]): Promise<{
|
|
245
|
+
queued: number;
|
|
246
|
+
alreadyActive: number;
|
|
247
|
+
cleanedNonFiles: number;
|
|
248
|
+
resumed: boolean;
|
|
249
|
+
rejected: number;
|
|
250
|
+
}>;
|
|
251
|
+
/** 上是否已?queued/running ?parse/fetch 任务 */
|
|
252
|
+
private hasActiveParseJob;
|
|
253
|
+
/**
|
|
254
|
+
* 级任?文件UI 跟踪重做完成?
|
|
255
|
+
*
|
|
256
|
+
* @param sourceId - ?id
|
|
257
|
+
* @param paths - 列表
|
|
258
|
+
*/
|
|
259
|
+
jobStateForPaths(sourceId: string, paths: string[]): Array<{
|
|
260
|
+
path: string;
|
|
261
|
+
jobsActive: number;
|
|
262
|
+
fileStatus: string | null;
|
|
263
|
+
chunkCount: number;
|
|
264
|
+
error: string | null;
|
|
265
|
+
exists: boolean;
|
|
266
|
+
}>;
|
|
267
|
+
/**
|
|
268
|
+
* 按筛选批量重做(当前列表?status/ext/q 条件?
|
|
269
|
+
*
|
|
270
|
+
* @param sourceId - ?id
|
|
271
|
+
* @param opts - 与文件列表筛选一?
|
|
272
|
+
* @returns 入队条数
|
|
273
|
+
*/
|
|
274
|
+
reprocessByFilter(sourceId: string, opts?: {
|
|
275
|
+
status?: 'indexed' | 'skipped' | 'error' | 'all';
|
|
276
|
+
ext?: string;
|
|
277
|
+
q?: string;
|
|
278
|
+
}): Promise<{
|
|
279
|
+
queued: number;
|
|
280
|
+
alreadyActive: number;
|
|
281
|
+
cleanedNonFiles: number;
|
|
282
|
+
resumed: boolean;
|
|
283
|
+
rejected: number;
|
|
284
|
+
}>;
|
|
132
285
|
/**
|
|
133
286
|
* 处理 fs 变更(debounce 后入队)
|
|
134
287
|
*/
|
|
135
288
|
handleFsChange(sourceId: string, filePath: string, event: 'change' | 'rename'): void;
|
|
136
289
|
/**
|
|
137
|
-
*
|
|
290
|
+
* watch(幂等)
|
|
138
291
|
*/
|
|
139
292
|
startWatch(sourceId: string): void;
|
|
140
293
|
stopWatch(sourceId: string): void;
|
|
141
294
|
/**
|
|
142
|
-
*
|
|
295
|
+
* 等待队列排空(测?管理?
|
|
143
296
|
*/
|
|
144
297
|
idle(timeoutMs?: number): Promise<void>;
|
|
145
298
|
dispose(): void;
|
|
146
299
|
/**
|
|
147
|
-
*
|
|
300
|
+
* 手动索引任务(业务完整:停收 + 清队 + 打断在跑?
|
|
301
|
+
*
|
|
302
|
+
* @param opts.sourceId - 仅中源;省略则全部源
|
|
303
|
+
* @returns 取消
|
|
304
|
+
*/
|
|
305
|
+
abortJobs(opts?: {
|
|
306
|
+
sourceId?: string;
|
|
307
|
+
}): {
|
|
308
|
+
cancelledQueued: number;
|
|
309
|
+
abortedRunning: number;
|
|
310
|
+
runningJobs: number;
|
|
311
|
+
};
|
|
312
|
+
/**
|
|
313
|
+
* /继续索引(相轻)
|
|
314
|
+
*
|
|
315
|
+
* - 清除该源?
|
|
316
|
+
* - ?cancelled 任务放回 queued
|
|
317
|
+
* - 若仍缺向量则排队 embed
|
|
318
|
+
* - 不重?walk 全量;需要全 reindex
|
|
319
|
+
*
|
|
320
|
+
* @param opts.sourceId - 仅恢源;省略则全部源
|
|
321
|
+
* @returns
|
|
322
|
+
*/
|
|
323
|
+
resumeJobs(opts?: {
|
|
324
|
+
sourceId?: string;
|
|
325
|
+
}): {
|
|
326
|
+
restoredCancelled: number;
|
|
327
|
+
embedQueued: number;
|
|
328
|
+
};
|
|
329
|
+
/** 的未(resume / supersede 后用;清 DB 态) */
|
|
330
|
+
private beginAbortEpoch;
|
|
331
|
+
/**
|
|
332
|
+
* 重建前作废本源未完成任务(supersede?
|
|
333
|
+
*
|
|
334
|
+
* - queued ?cancelled(superseded_by_reindex?
|
|
335
|
+
* - running 打断(worker/出)
|
|
336
|
+
* - 随后 beginAbortEpoch,本建可继续?
|
|
337
|
+
*/
|
|
338
|
+
private supersedeSourceWork;
|
|
339
|
+
private ensureParseCoverage;
|
|
340
|
+
/** watch 突发/目录事件后:尽快做一次缺口补扫(默认 3s,不等 60s 节流) */
|
|
341
|
+
private requestGapScan;
|
|
342
|
+
/**
|
|
343
|
+
* 磁盘↔索齐:删掉已不存在的文件(?chunks/embeddings?
|
|
344
|
+
*
|
|
345
|
+
* 源用?discover walk ?keep 集(;失回并?stat?
|
|
346
|
+
* embed 前调避免给已移出文件的残 chunks 上向?
|
|
347
|
+
*
|
|
348
|
+
* @param sourceId - 知识?id
|
|
349
|
+
* @returns 删除的文件数
|
|
350
|
+
*/
|
|
351
|
+
pruneMissingOnDisk(sourceId: string): Promise<number>;
|
|
352
|
+
/** 逐文?stat(file ?/ walk 失败);限流 + 单文件 removeFile(勿对每个 gone 做 removePathTree LIKE) */
|
|
353
|
+
private pruneByStatScan;
|
|
354
|
+
private abortSignalFor;
|
|
355
|
+
private isAborted;
|
|
356
|
+
/**
|
|
357
|
+
* /继续 控制数(互斥按钮
|
|
358
|
+
*
|
|
359
|
+
* @param sourceId - ?id
|
|
360
|
+
* @returns 队列计数 + 已中?+ 仍缺向量
|
|
361
|
+
*/
|
|
362
|
+
jobControlState(sourceId: string): KnowledgeJobControlState;
|
|
363
|
+
/**
|
|
364
|
+
* 任务对账(看门狗?不依赖单?ensureEmbedJob
|
|
365
|
+
*
|
|
366
|
+
* 解决「续去重吞掉 / 进程 / partial」等静默停摆?
|
|
367
|
+
* 1. 回收超时 running(崩?孤儿?
|
|
368
|
+
* 2. 有缺向量且无 active embed ?排队
|
|
369
|
+
* 3. 无队缺口 ??coverage/status
|
|
370
|
+
*/
|
|
371
|
+
startReconciler(intervalMs?: number): void;
|
|
372
|
+
stopReconciler(): void;
|
|
373
|
+
/** 对账入口(也工调 */
|
|
374
|
+
reconcileJobs(): Promise<void>;
|
|
375
|
+
/** ?job 清理(done/failed/cancelled 超过保留期) */
|
|
376
|
+
private cleanupTerminalJobs;
|
|
377
|
+
/** running 超时(无 heartbeat)→ 收回 queued次失败的直接?failed,防毒文 */
|
|
378
|
+
private reclaimStaleRunningJobs;
|
|
379
|
+
private countActiveJobs;
|
|
380
|
+
private hasRunningKind;
|
|
381
|
+
/**
|
|
382
|
+
* poll 调度(幂等);仅处理 sync.strategy=poll ?enabled 的源
|
|
148
383
|
*/
|
|
149
384
|
startPolling(): void;
|
|
150
385
|
stopPolling(): void;
|
|
151
386
|
/**
|
|
152
|
-
*
|
|
387
|
+
* 到期 poll 源并入队刷新(内容未变由 content-hash / 304 ?
|
|
153
388
|
*
|
|
154
|
-
* @returns
|
|
389
|
+
* @returns 实际触发?sourceId 列表
|
|
155
390
|
*/
|
|
156
391
|
pollDueSources(): Promise<string[]>;
|
|
157
392
|
private hasQueuedJobs;
|
|
393
|
+
/**
|
|
394
|
+
* 入队(同步调度用;失败不抛——DB 已关/中止时静默丢弃)。
|
|
395
|
+
* 需要计数的路径请 `await enqueueStrict`。
|
|
396
|
+
*/
|
|
158
397
|
private enqueue;
|
|
398
|
+
/** 入队并绑定真实 file_id(JobQueue 去重 / purge 清 job / abort 共享 File 守卫)。
|
|
399
|
+
*
|
|
400
|
+
* 共享 File:同 file_id 已有 parse 在途时只补 Membership,禁止双 parse。
|
|
401
|
+
*/
|
|
402
|
+
private enqueueStrict;
|
|
403
|
+
/** parse/fetch 成功后回填 job.file_id(新建 File 的首条 job 入队时尚无 id) */
|
|
404
|
+
private bindJobFileId;
|
|
405
|
+
/** job 绑定 file_id:membership → identity;本地文件可 identify */
|
|
406
|
+
private resolveJobFileId;
|
|
159
407
|
private kick;
|
|
160
408
|
private drain;
|
|
409
|
+
/** 文档抽取单独限流:多份 PDF/Office 同时抽会打爆内存与 Engine */
|
|
410
|
+
private isDocParseJob;
|
|
411
|
+
private acceptParseSlot;
|
|
412
|
+
private countQueuedKinds;
|
|
161
413
|
private claimJob;
|
|
162
414
|
private runJob;
|
|
163
415
|
/**
|
|
164
|
-
*
|
|
416
|
+
* 带截止时间的消执行:超时?abort signal,工作侧必须在副作用?
|
|
417
|
+
* 再用 Promise.race 包不消的 Promise会在超时后留下僵尸写?
|
|
418
|
+
*/
|
|
419
|
+
private withTimeout;
|
|
420
|
+
/**
|
|
421
|
+
* 外源单文?job:按逻辑?fetch + 索引
|
|
165
422
|
*/
|
|
166
423
|
private fetchDocJob;
|
|
167
424
|
private parseOne;
|
|
168
425
|
/**
|
|
169
|
-
* PDF/Office:DocumentPort
|
|
170
|
-
* 新鲜度以原件字节 hash
|
|
426
|
+
* PDF/Office:DocumentPort 抽取?Markdown 后走 markdownAdapter 切块?
|
|
427
|
+
* 新鲜度以原件字节 hash 为准?
|
|
171
428
|
*
|
|
172
|
-
* @param sourceId -
|
|
173
|
-
* @param filePath -
|
|
174
|
-
* @returns
|
|
429
|
+
* @param sourceId - 知识?id
|
|
430
|
+
* @param filePath - 文件
|
|
431
|
+
* @returns 成功入索?
|
|
175
432
|
*/
|
|
176
433
|
private parseDocumentFile;
|
|
434
|
+
/** worker 抽取 + 流式 hash + 切块;注入 port 时(测试)由调用侧流式 hash */
|
|
435
|
+
private extractDocument;
|
|
436
|
+
/** 纯文本切块 + ftsToks(虚拟文档 / port 回退共用) */
|
|
437
|
+
private chunkTextInWorker;
|
|
177
438
|
/**
|
|
178
|
-
* Phase B:嵌入待处理 chunk
|
|
439
|
+
* Phase B:嵌入待处理 chunk
|
|
440
|
+
*
|
|
441
|
+
* ?job 内按 embedConcurrency ****多批调用 API?
|
|
442
|
+
* 失败/超时?re-enqueue,避?job ?done ?coverage 永久卡住?
|
|
179
443
|
*/
|
|
180
444
|
private embedPending;
|
|
445
|
+
private recordEmbedSecretHit;
|
|
446
|
+
/** 仍有缺向量则排队下一?embed_source(须在当?job ?done 之后调用?*/
|
|
447
|
+
private ensureEmbedJob;
|
|
448
|
+
private heartbeatJob;
|
|
449
|
+
/** parse 完成不立刻刷 coverage(O(N) 写放大);标脏,drain/reconcile 时合并刷?*/
|
|
450
|
+
private coverageDirty;
|
|
451
|
+
private markCoverageDirty;
|
|
452
|
+
private flushCoverageDirty;
|
|
453
|
+
private lastCoverageRefreshAt;
|
|
181
454
|
private refreshCoverage;
|
|
182
455
|
private emitProgress;
|
|
183
|
-
/**
|
|
456
|
+
/** 磁盘水位(仅,不务) */
|
|
184
457
|
private warnDiskWatermark;
|
|
185
458
|
}
|