dsh-data-domain-knowledge 0.1.0-alpha.9 → 0.1.1-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -567
- package/cordis.patch.yml +5 -418
- package/eval/README.md +19 -8
- package/eval/benchmark-extraction-format.mjs +20 -0
- package/eval/evaluate-retrieval.mjs +17 -8
- package/eval/queries.example.jsonl +1 -0
- package/lib/client.js +4181 -3239
- package/lib/contract-envelope.d.ts +91 -0
- package/lib/contract-envelope.js +106 -0
- package/lib/contract-layout.d.ts +130 -0
- package/lib/contract-layout.js +138 -0
- package/lib/contract-refs.d.ts +39 -0
- package/lib/contract-refs.js +9 -0
- package/lib/contract-version.d.ts +311 -0
- package/lib/contract-version.js +270 -0
- package/lib/contract-wire.d.ts +147 -0
- package/lib/contract-wire.js +99 -0
- package/lib/contracts.d.ts +12 -0
- package/lib/contracts.js +12 -0
- package/lib/doc-page.js +2 -2
- package/lib/pages.js +3329 -1296
- package/lib/types/agent-behavior-rules.d.ts +10 -0
- package/lib/types/agent-behavior-rules.d.ts.map +1 -0
- package/lib/types/agent-behavior-rules.js +28 -0
- package/lib/types/agent-behavior-rules.js.map +1 -0
- package/lib/types/axis-registry.d.ts +37 -5
- package/lib/types/axis-registry.d.ts.map +1 -1
- package/lib/types/axis-registry.js +234 -26
- package/lib/types/axis-registry.js.map +1 -1
- package/lib/types/card-organize.d.ts +27 -12
- package/lib/types/card-organize.d.ts.map +1 -1
- package/lib/types/card-organize.js +58 -73
- package/lib/types/card-organize.js.map +1 -1
- package/lib/types/compiler/embedding-identity.d.ts +3 -0
- package/lib/types/compiler/embedding-identity.d.ts.map +1 -0
- package/lib/types/compiler/embedding-identity.js +24 -0
- package/lib/types/compiler/embedding-identity.js.map +1 -0
- package/lib/types/compiler/snapshot.d.ts +16 -0
- package/lib/types/compiler/snapshot.d.ts.map +1 -0
- package/lib/types/compiler/snapshot.js +2 -0
- package/lib/types/compiler/snapshot.js.map +1 -0
- package/lib/types/config-schema.d.ts +6 -3
- package/lib/types/config-schema.d.ts.map +1 -1
- package/lib/types/config-schema.js +6 -2
- package/lib/types/config-schema.js.map +1 -1
- package/lib/types/convert/host.d.ts +6 -5
- package/lib/types/convert/host.d.ts.map +1 -1
- package/lib/types/convert/host.js +58 -18
- package/lib/types/convert/host.js.map +1 -1
- package/lib/types/convert/ingest.d.ts +10 -1
- package/lib/types/convert/ingest.d.ts.map +1 -1
- package/lib/types/convert/ingest.js +68 -9
- package/lib/types/convert/ingest.js.map +1 -1
- package/lib/types/derive-contract.d.ts +3 -0
- package/lib/types/derive-contract.d.ts.map +1 -0
- package/lib/types/derive-contract.js +15 -0
- package/lib/types/derive-contract.js.map +1 -0
- package/lib/types/diagnosis/cards.d.ts +7 -3
- package/lib/types/diagnosis/cards.d.ts.map +1 -1
- package/lib/types/diagnosis/cards.js +13 -8
- package/lib/types/diagnosis/cards.js.map +1 -1
- package/lib/types/diagnosis/run.d.ts +4 -13
- package/lib/types/diagnosis/run.d.ts.map +1 -1
- package/lib/types/diagnosis/run.js +26 -16
- package/lib/types/diagnosis/run.js.map +1 -1
- package/lib/types/diagnosis/schema.js +2 -2
- package/lib/types/diagnosis/schema.js.map +1 -1
- package/lib/types/diagnosis/seed-source.d.ts +1 -1
- package/lib/types/diagnosis/seed-source.d.ts.map +1 -1
- package/lib/types/diagnosis/seed-source.js +5 -4
- package/lib/types/diagnosis/seed-source.js.map +1 -1
- package/lib/types/diagnosis/seed.d.ts +4 -4
- package/lib/types/diagnosis/seed.d.ts.map +1 -1
- package/lib/types/diagnosis/seed.js +6 -8
- package/lib/types/diagnosis/seed.js.map +1 -1
- package/lib/types/diagnosis/store.d.ts +24 -18
- package/lib/types/diagnosis/store.d.ts.map +1 -1
- package/lib/types/diagnosis/store.js +130 -65
- package/lib/types/diagnosis/store.js.map +1 -1
- package/lib/types/diagnosis/types.d.ts +47 -15
- package/lib/types/diagnosis/types.d.ts.map +1 -1
- package/lib/types/diagnosis/types.js.map +1 -1
- package/lib/types/environment/dsh-python.d.ts +77 -0
- package/lib/types/environment/dsh-python.d.ts.map +1 -0
- package/lib/types/environment/dsh-python.js +139 -0
- package/lib/types/environment/dsh-python.js.map +1 -0
- package/lib/types/environment/install.d.ts +15 -0
- package/lib/types/environment/install.d.ts.map +1 -0
- package/lib/types/environment/install.js +297 -0
- package/lib/types/environment/install.js.map +1 -0
- package/lib/types/environment/manifest.d.ts +208 -0
- package/lib/types/environment/manifest.d.ts.map +1 -0
- package/lib/types/environment/manifest.js +26 -0
- package/lib/types/environment/manifest.js.map +1 -0
- package/lib/types/environment/model-catalog.d.ts +223 -0
- package/lib/types/environment/model-catalog.d.ts.map +1 -0
- package/lib/types/environment/model-catalog.js +129 -0
- package/lib/types/environment/model-catalog.js.map +1 -0
- package/lib/types/environment/process.d.ts +10 -0
- package/lib/types/environment/process.d.ts.map +1 -0
- package/lib/types/environment/process.js +54 -0
- package/lib/types/environment/process.js.map +1 -0
- package/lib/types/environment/selection.d.ts +9 -0
- package/lib/types/environment/selection.d.ts.map +1 -0
- package/lib/types/environment/selection.js +29 -0
- package/lib/types/environment/selection.js.map +1 -0
- package/lib/types/environment/service.d.ts +64 -0
- package/lib/types/environment/service.d.ts.map +1 -0
- package/lib/types/environment/service.js +424 -0
- package/lib/types/environment/service.js.map +1 -0
- package/lib/types/environment/types.d.ts +56 -0
- package/lib/types/environment/types.d.ts.map +1 -0
- package/lib/types/environment/types.js +2 -0
- package/lib/types/environment/types.js.map +1 -0
- package/lib/types/events.d.ts +76 -0
- package/lib/types/events.d.ts.map +1 -0
- package/lib/types/events.js +52 -0
- package/lib/types/events.js.map +1 -0
- package/lib/types/extract-contract.d.ts +3 -0
- package/lib/types/extract-contract.d.ts.map +1 -0
- package/lib/types/extract-contract.js +41 -0
- package/lib/types/extract-contract.js.map +1 -0
- package/lib/types/extract-model.d.ts +5 -0
- package/lib/types/extract-model.d.ts.map +1 -1
- package/lib/types/extract-model.js.map +1 -1
- package/lib/types/extract-table-contract.d.ts +3 -0
- package/lib/types/extract-table-contract.d.ts.map +1 -0
- package/lib/types/extract-table-contract.js +27 -0
- package/lib/types/extract-table-contract.js.map +1 -0
- package/lib/types/extract-table.d.ts +8 -0
- package/lib/types/extract-table.d.ts.map +1 -0
- package/lib/types/extract-table.js +158 -0
- package/lib/types/extract-table.js.map +1 -0
- package/lib/types/extraction-runs.d.ts +30 -6
- package/lib/types/extraction-runs.d.ts.map +1 -1
- package/lib/types/extraction-runs.js +69 -8
- package/lib/types/extraction-runs.js.map +1 -1
- package/lib/types/fs-atomic.d.ts +0 -7
- package/lib/types/fs-atomic.d.ts.map +1 -1
- package/lib/types/fs-atomic.js +17 -2
- package/lib/types/fs-atomic.js.map +1 -1
- package/lib/types/images.d.ts +26 -0
- package/lib/types/images.d.ts.map +1 -0
- package/lib/types/images.js +196 -0
- package/lib/types/images.js.map +1 -0
- package/lib/types/index.d.ts +2 -2
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +598 -209
- package/lib/types/index.js.map +1 -1
- package/lib/types/intent/engine.d.ts +12 -4
- package/lib/types/intent/engine.d.ts.map +1 -1
- package/lib/types/intent/engine.js +43 -5
- package/lib/types/intent/engine.js.map +1 -1
- package/lib/types/intent/host.d.ts +5 -8
- package/lib/types/intent/host.d.ts.map +1 -1
- package/lib/types/intent/host.js +56 -11
- package/lib/types/intent/host.js.map +1 -1
- package/lib/types/intent/scope.d.ts +25 -0
- package/lib/types/intent/scope.d.ts.map +1 -0
- package/lib/types/intent/scope.js +116 -0
- package/lib/types/intent/scope.js.map +1 -0
- package/lib/types/intent/tools.d.ts +24 -20
- package/lib/types/intent/tools.d.ts.map +1 -1
- package/lib/types/intent/tools.js +65 -23
- package/lib/types/intent/tools.js.map +1 -1
- package/lib/types/intent/types.d.ts +11 -4
- package/lib/types/intent/types.d.ts.map +1 -1
- package/lib/types/knowledge-bases.d.ts +11 -5
- package/lib/types/knowledge-bases.d.ts.map +1 -1
- package/lib/types/knowledge-bases.js +52 -20
- package/lib/types/knowledge-bases.js.map +1 -1
- package/lib/types/knowledge-contract.d.ts +32 -0
- package/lib/types/knowledge-contract.d.ts.map +1 -0
- package/lib/types/knowledge-contract.js +116 -0
- package/lib/types/knowledge-contract.js.map +1 -0
- package/lib/types/knowledge.d.ts +31 -9
- package/lib/types/knowledge.d.ts.map +1 -1
- package/lib/types/knowledge.js +137 -60
- package/lib/types/knowledge.js.map +1 -1
- package/lib/types/layout.d.ts +10 -29
- package/lib/types/layout.d.ts.map +1 -1
- package/lib/types/layout.js +17 -36
- package/lib/types/layout.js.map +1 -1
- package/lib/types/materialize-snapshot.d.ts +49 -0
- package/lib/types/materialize-snapshot.d.ts.map +1 -0
- package/lib/types/materialize-snapshot.js +2 -0
- package/lib/types/materialize-snapshot.js.map +1 -0
- package/lib/types/migrate.d.ts +11 -0
- package/lib/types/migrate.d.ts.map +1 -1
- package/lib/types/migrate.js +83 -2
- package/lib/types/migrate.js.map +1 -1
- package/lib/types/model-text.d.ts +58 -1
- package/lib/types/model-text.d.ts.map +1 -1
- package/lib/types/model-text.js +85 -30
- package/lib/types/model-text.js.map +1 -1
- package/lib/types/oplog.d.ts.map +1 -1
- package/lib/types/oplog.js +7 -0
- package/lib/types/oplog.js.map +1 -1
- package/lib/types/organize-contract.d.ts +18 -0
- package/lib/types/organize-contract.d.ts.map +1 -0
- package/lib/types/organize-contract.js +82 -0
- package/lib/types/organize-contract.js.map +1 -0
- package/lib/types/pipeline/engine.d.ts +66 -0
- package/lib/types/pipeline/engine.d.ts.map +1 -0
- package/lib/types/pipeline/engine.js +352 -0
- package/lib/types/pipeline/engine.js.map +1 -0
- package/lib/types/pipeline/store.d.ts +73 -0
- package/lib/types/pipeline/store.d.ts.map +1 -0
- package/lib/types/pipeline/store.js +143 -0
- package/lib/types/pipeline/store.js.map +1 -0
- package/lib/types/pipeline/types.d.ts +102 -0
- package/lib/types/pipeline/types.d.ts.map +1 -0
- package/lib/types/pipeline/types.js +5 -0
- package/lib/types/pipeline/types.js.map +1 -0
- package/lib/types/present.d.ts +2 -0
- package/lib/types/present.d.ts.map +1 -1
- package/lib/types/present.js +9 -4
- package/lib/types/present.js.map +1 -1
- package/lib/types/preset-ids.d.ts +4 -0
- package/lib/types/preset-ids.d.ts.map +1 -0
- package/lib/types/preset-ids.js +6 -0
- package/lib/types/preset-ids.js.map +1 -0
- package/lib/types/query-analysis.d.ts +13 -1
- package/lib/types/query-analysis.d.ts.map +1 -1
- package/lib/types/query-analysis.js +62 -6
- package/lib/types/query-analysis.js.map +1 -1
- package/lib/types/recall-rank.d.ts +3 -3
- package/lib/types/recall-rank.d.ts.map +1 -1
- package/lib/types/recall-rank.js +4 -4
- package/lib/types/recall-rank.js.map +1 -1
- package/lib/types/retention.d.ts +39 -0
- package/lib/types/retention.d.ts.map +1 -0
- package/lib/types/retention.js +42 -0
- package/lib/types/retention.js.map +1 -0
- package/lib/types/retrieval/vector-index.d.ts +32 -0
- package/lib/types/retrieval/vector-index.d.ts.map +1 -0
- package/lib/types/retrieval/vector-index.js +15 -0
- package/lib/types/retrieval/vector-index.js.map +1 -0
- package/lib/types/retrieval/visibility.d.ts +6 -0
- package/lib/types/retrieval/visibility.d.ts.map +1 -0
- package/lib/types/retrieval/visibility.js +11 -0
- package/lib/types/retrieval/visibility.js.map +1 -0
- package/lib/types/runtime.d.ts +272 -27
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/runtime.js +2751 -537
- package/lib/types/runtime.js.map +1 -1
- package/lib/types/settings-schema.d.ts +17 -15
- package/lib/types/settings-schema.d.ts.map +1 -1
- package/lib/types/settings-schema.js +50 -47
- package/lib/types/settings-schema.js.map +1 -1
- package/lib/types/settings-volatile.d.ts +4 -4
- package/lib/types/settings-volatile.js +4 -4
- package/lib/types/source-evidence.d.ts +14 -0
- package/lib/types/source-evidence.d.ts.map +1 -0
- package/lib/types/source-evidence.js +36 -0
- package/lib/types/source-evidence.js.map +1 -0
- package/lib/types/store/axis-references.d.ts +5 -0
- package/lib/types/store/axis-references.d.ts.map +1 -0
- package/lib/types/store/axis-references.js +106 -0
- package/lib/types/store/axis-references.js.map +1 -0
- package/lib/types/store/derived.d.ts +12 -1
- package/lib/types/store/derived.d.ts.map +1 -1
- package/lib/types/store/derived.js +71 -12
- package/lib/types/store/derived.js.map +1 -1
- package/lib/types/store/knowledge-db.d.ts +31 -1
- package/lib/types/store/knowledge-db.d.ts.map +1 -1
- package/lib/types/store/knowledge-db.js +252 -18
- package/lib/types/store/knowledge-db.js.map +1 -1
- package/lib/types/store/migrate.d.ts.map +1 -1
- package/lib/types/store/migrate.js +17 -6
- package/lib/types/store/migrate.js.map +1 -1
- package/lib/types/trajectory-types.d.ts +172 -0
- package/lib/types/trajectory-types.d.ts.map +1 -0
- package/lib/types/trajectory-types.js +36 -0
- package/lib/types/trajectory-types.js.map +1 -0
- package/lib/types/types.d.ts +59 -27
- package/lib/types/types.d.ts.map +1 -1
- package/lib/types/types.js +6 -15
- package/lib/types/types.js.map +1 -1
- package/lib/types/wiki-finalize.d.ts +16 -0
- package/lib/types/wiki-finalize.d.ts.map +1 -1
- package/lib/types/wiki-finalize.js +26 -0
- package/lib/types/wiki-finalize.js.map +1 -1
- package/lib/types/wiki-materialize.d.ts +4 -0
- package/lib/types/wiki-materialize.d.ts.map +1 -1
- package/lib/types/wiki-materialize.js +21 -7
- package/lib/types/wiki-materialize.js.map +1 -1
- package/lib/types/wiki-trial-types.d.ts +87 -0
- package/lib/types/wiki-trial-types.d.ts.map +1 -1
- package/lib/types/wiki-trial-types.js +0 -16
- package/lib/types/wiki-trial-types.js.map +1 -1
- package/lib/types/wiki-trial.d.ts +40 -16
- package/lib/types/wiki-trial.d.ts.map +1 -1
- package/lib/types/wiki-trial.js +337 -224
- package/lib/types/wiki-trial.js.map +1 -1
- package/package.json +29 -20
- package/python/prepare-models.py +19 -0
- package/python/requirements.txt +8 -6
- package/scripts/doctor.mjs +13 -19
- package/scripts/environment.mjs +29 -0
- package/scripts/setup-embedding.sh +15 -5
- package/src/agent-behavior-rules.ts +37 -0
- package/src/axis-registry.ts +234 -27
- package/src/card-organize.ts +69 -75
- package/src/client/AgentBehaviorRulesCard.tsx +65 -0
- package/src/client/AttributionEventGuide.tsx +41 -39
- package/src/client/AxisRegistrySection.tsx +204 -164
- package/src/client/CompileDialogs.tsx +44 -29
- package/src/client/DeleteDocumentDialog.tsx +36 -5
- package/src/client/DirectoryPicker.tsx +0 -10
- package/src/client/EnvironmentSetup.tsx +243 -0
- package/src/client/ExpansionGraph.tsx +13 -7
- package/src/client/ExtractionTasksPage.tsx +578 -0
- package/src/client/GuideArchitectureCanvas.tsx +470 -0
- package/src/client/ImportFilesDialog.tsx +2 -13
- package/src/client/IntentLogic.tsx +2 -2
- package/src/client/IntentTab.tsx +118 -47
- package/src/client/KnowledgeBaseTab.tsx +277 -197
- package/src/client/KnowledgeLayerSection.tsx +24 -6
- package/src/client/KnowledgePanel.tsx +58 -93
- package/src/client/KnowledgeSettingsDrawer.tsx +90 -0
- package/src/client/ListPager.tsx +15 -5
- package/src/client/MaterializeTrajectoryView.tsx +124 -0
- package/src/client/NodeSettings.tsx +210 -89
- package/src/client/NormalizeTrajectoryView.tsx +136 -0
- package/src/client/PipelineNotice.tsx +26 -0
- package/src/client/PreviewSections.tsx +118 -51
- package/src/client/RecallPanel.tsx +89 -85
- package/src/client/TabGuide.tsx +159 -184
- package/src/client/TrialReportView.tsx +391 -214
- package/src/client/TypeDefinitionDrawer.tsx +150 -23
- package/src/client/WikiCardPreview.tsx +62 -16
- package/src/client/WikiFlowPanel.tsx +4 -4
- package/src/client/arch/NodeLogicPanel.tsx +3 -4
- package/src/client/arch/model.ts +6 -9
- package/src/client/architecture.tsx +23 -33
- package/src/client/attribution/AttributionGraph.tsx +10 -6
- package/src/client/attribution/AttributionTab.tsx +136 -84
- package/src/client/attribution/CaseView.tsx +3 -1
- package/src/client/attribution/EventView.tsx +108 -0
- package/src/client/attribution/RunView.tsx +2 -2
- package/src/client/attribution/StageChrome.tsx +34 -3
- package/src/client/attribution/ViewSwitch.tsx +20 -5
- package/src/client/attribution/graph-options.ts +47 -39
- package/src/client/attribution/model.ts +17 -10
- package/src/client/config-scope.ts +3 -3
- package/src/client/graph/palette.ts +165 -0
- package/src/client/index.ts +49 -42
- package/src/client/pages.css +459 -135
- package/src/client/pages.tsx +37 -9
- package/src/client/problem-domains.ts +1 -1
- package/src/client/session-remote.ts +9 -0
- package/src/client/styles.css +205 -62
- package/src/client/wiki-trial.ts +44 -3
- package/src/compiler/embedding-identity.ts +21 -0
- package/src/compiler/snapshot.ts +17 -0
- package/src/config-schema.ts +8 -3
- package/src/convert/host.ts +56 -21
- package/src/convert/ingest.ts +74 -10
- package/src/derive-contract.ts +14 -0
- package/src/diagnosis/cards.ts +13 -8
- package/src/diagnosis/run.ts +27 -17
- package/src/diagnosis/schema.ts +2 -2
- package/src/diagnosis/seed-source.ts +5 -4
- package/src/diagnosis/seed.ts +6 -8
- package/src/diagnosis/store.ts +138 -69
- package/src/diagnosis/types.ts +34 -15
- package/src/environment/dsh-python.ts +156 -0
- package/src/environment/install.ts +244 -0
- package/src/environment/manifest.ts +28 -0
- package/src/environment/model-catalog.ts +147 -0
- package/src/environment/process.ts +54 -0
- package/src/environment/selection.ts +29 -0
- package/src/environment/service.ts +348 -0
- package/src/environment/types.ts +45 -0
- package/src/events.ts +108 -0
- package/src/extract-contract.ts +42 -0
- package/src/extract-model.ts +10 -1
- package/src/extract-table-contract.ts +26 -0
- package/src/extract-table.ts +132 -0
- package/src/extraction-runs.ts +70 -11
- package/src/fs-atomic.ts +19 -2
- package/src/images.ts +203 -0
- package/src/index.ts +553 -205
- package/src/intent/README.md +8 -6
- package/src/intent/engine.ts +44 -6
- package/src/intent/host.ts +59 -18
- package/src/intent/scope.ts +125 -0
- package/src/intent/tools.ts +70 -28
- package/src/intent/types.ts +11 -4
- package/src/knowledge-bases.ts +56 -20
- package/src/knowledge-contract.ts +114 -0
- package/src/knowledge.ts +138 -69
- package/src/layout.ts +17 -37
- package/src/materialize-snapshot.ts +46 -0
- package/src/migrate.ts +75 -2
- package/src/model-text.ts +109 -4
- package/src/oplog.ts +7 -0
- package/src/organize-contract.ts +83 -0
- package/src/pipeline/engine.ts +323 -0
- package/src/pipeline/store.ts +147 -0
- package/src/pipeline/types.ts +93 -0
- package/src/present.ts +10 -4
- package/src/preset-ids.ts +6 -0
- package/src/python/embedding_worker.py +66 -2
- package/src/query-analysis.ts +68 -8
- package/src/recall-rank.ts +5 -5
- package/src/retention.ts +76 -0
- package/src/retrieval/vector-index.ts +45 -0
- package/src/retrieval/visibility.ts +14 -0
- package/src/runtime.ts +2708 -567
- package/src/settings-schema.ts +11 -8
- package/src/settings-volatile.ts +4 -4
- package/src/source-evidence.ts +37 -0
- package/src/store/axis-references.ts +96 -0
- package/src/store/derived.ts +70 -12
- package/src/store/knowledge-db.ts +248 -17
- package/src/store/migrate.ts +16 -6
- package/src/trajectory-types.ts +142 -0
- package/src/types.ts +61 -35
- package/src/wiki-finalize.ts +28 -0
- package/src/wiki-materialize.ts +25 -6
- package/src/wiki-trial-types.ts +83 -1
- package/src/wiki-trial.ts +357 -185
- package/lib/types/intent/preset-tools.d.ts +0 -37
- package/lib/types/intent/preset-tools.d.ts.map +0 -1
- package/lib/types/intent/preset-tools.js +0 -45
- package/lib/types/intent/preset-tools.js.map +0 -1
- package/preset/tita-data-agent/agent.cordis.yml +0 -394
- package/preset/tita-data-agent/preset.yml +0 -3
- package/scripts/install-agent-preset.mjs +0 -92
- package/scripts/sync-agent-preset.mjs +0 -188
- package/src/client/AgentPresetCard.tsx +0 -247
- package/src/client/agent-preset.ts +0 -121
- package/src/client/convert/ConvertPanel.tsx +0 -583
- package/src/client/convert/styles.css +0 -138
- package/src/client/convert/styles.ts +0 -49
- package/src/intent/preset-tools.ts +0 -48
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
2
|
+
import { existsSync } from 'node:fs'
|
|
3
|
+
import { chmod, copyFile, mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises'
|
|
4
|
+
import { homedir } from 'node:os'
|
|
5
|
+
import { dirname, join } from 'node:path'
|
|
6
|
+
import { fileURLToPath } from 'node:url'
|
|
7
|
+
import { modelsForSelection, UV_VERSION } from './manifest.js'
|
|
8
|
+
import { DEFAULT_MODEL_SELECTION, type ModelSelection } from './model-catalog.js'
|
|
9
|
+
import { writeModelSelection } from './selection.js'
|
|
10
|
+
import { runProcess, type ProcessOptions } from './process.js'
|
|
11
|
+
import { hasHardenedRuntime, needsManagedInterpreter, pickBasePython, pythonSourceLabel } from './dsh-python.js'
|
|
12
|
+
|
|
13
|
+
export const requirementsFile = fileURLToPath(new URL('../../../python/requirements.txt', import.meta.url))
|
|
14
|
+
export const setupScript = fileURLToPath(new URL('../../../python/prepare-models.py', import.meta.url))
|
|
15
|
+
export function managedPython(root: string): string { return process.platform === 'win32' ? join(root, 'python', 'Scripts', 'python.exe') : join(root, 'python', 'bin', 'python') }
|
|
16
|
+
|
|
17
|
+
/** 只问次版本:判断的是「能不能装这套依赖」,不是补丁号。 */
|
|
18
|
+
const PYTHON_VERSION_SCRIPT = 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'
|
|
19
|
+
/** 推理依赖支持的解释器区间(torch 的 wheel 覆盖到 3.13)。 */
|
|
20
|
+
const SUPPORTED_PYTHON = /^3\.(10|11|12|13)$/
|
|
21
|
+
/** 走 uv 管理解释器时请求的版本:与桌面端自带的那份对齐,避免两套行为。 */
|
|
22
|
+
const MANAGED_PYTHON = '3.12'
|
|
23
|
+
|
|
24
|
+
async function download(url: string, signal: AbortSignal): Promise<Buffer> {
|
|
25
|
+
const response = await fetch(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(120_000)]) })
|
|
26
|
+
if (!response.ok) throw new Error(`下载失败(HTTP ${response.status}):${url}`)
|
|
27
|
+
return Buffer.from(await response.arrayBuffer())
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async function findExecutable(root: string, name: string): Promise<string | undefined> {
|
|
31
|
+
for (const entry of await readdir(root, { withFileTypes: true })) {
|
|
32
|
+
const path = join(root, entry.name)
|
|
33
|
+
if (entry.isFile() && entry.name === name) return path
|
|
34
|
+
if (entry.isDirectory()) { const found = await findExecutable(path, name); if (found) return found }
|
|
35
|
+
}
|
|
36
|
+
return undefined
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Prefer existing uv; otherwise install a checksummed release in this resource root. */
|
|
40
|
+
async function ensureUv(root: string, signal: AbortSignal, log: (line: string) => void, run: typeof runProcess): Promise<string> {
|
|
41
|
+
const name = process.platform === 'win32' ? 'uv.exe' : 'uv'
|
|
42
|
+
for (const candidate of [join(root, 'tools', name), 'uv', join(homedir(), '.local', 'bin', name), join(homedir(), '.cargo', 'bin', name)]) {
|
|
43
|
+
try { await run(candidate, ['--version'], { signal }); return candidate } catch { if (signal.aborted) throw new Error('ENVIRONMENT_CANCELLED') }
|
|
44
|
+
}
|
|
45
|
+
const arch = process.arch === 'arm64' ? 'aarch64' : process.arch === 'x64' ? 'x86_64' : undefined
|
|
46
|
+
const target = process.platform === 'darwin' ? 'apple-darwin' : process.platform === 'linux' ? 'unknown-linux-gnu' : process.platform === 'win32' ? 'pc-windows-msvc' : undefined
|
|
47
|
+
if (!arch || !target) throw new Error(`暂不支持自动安装:${process.platform}/${process.arch},请手动安装 uv 后重试`)
|
|
48
|
+
const extension = process.platform === 'win32' ? 'zip' : 'tar.gz'
|
|
49
|
+
const asset = `uv-${arch}-${target}.${extension}`
|
|
50
|
+
const url = `https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/${asset}`
|
|
51
|
+
log('正在下载 Python 安装工具 uv')
|
|
52
|
+
const checksum = (await download(`${url}.sha256`, signal)).toString().match(/[a-f0-9]{64}/i)?.[0]
|
|
53
|
+
const archive = await download(url, signal)
|
|
54
|
+
if (!checksum || createHash('sha256').update(archive).digest('hex') !== checksum.toLowerCase()) throw new Error('uv 下载文件校验失败,请重试')
|
|
55
|
+
const scratch = join(root, `.setup-uv-${randomUUID()}`)
|
|
56
|
+
await mkdir(scratch, { recursive: true })
|
|
57
|
+
try {
|
|
58
|
+
const path = join(scratch, asset)
|
|
59
|
+
await writeFile(path, archive)
|
|
60
|
+
if (process.platform === 'win32') {
|
|
61
|
+
const script = join(scratch, 'expand.ps1')
|
|
62
|
+
await writeFile(script, 'param([string]$Archive,[string]$Destination)\nExpand-Archive -LiteralPath $Archive -DestinationPath $Destination\n')
|
|
63
|
+
await run('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', script, path, join(scratch, 'expanded')], { signal })
|
|
64
|
+
} else await run('tar', ['-xzf', path, '-C', scratch], { signal })
|
|
65
|
+
const binary = await findExecutable(scratch, name)
|
|
66
|
+
if (!binary) throw new Error('uv 安装包缺少可执行文件')
|
|
67
|
+
const destination = join(root, 'tools', name)
|
|
68
|
+
await mkdir(dirname(destination), { recursive: true })
|
|
69
|
+
await copyFile(binary, destination)
|
|
70
|
+
if (process.platform !== 'win32') await chmod(destination, 0o755)
|
|
71
|
+
return destination
|
|
72
|
+
} finally { await rm(scratch, { recursive: true, force: true }) }
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* 一把「陈旧」锁可以被接管的时限。
|
|
77
|
+
*
|
|
78
|
+
* 单步安装超时是 30 分钟,整轮远小于这个上限,所以 6 小时之后还在的锁一定是崩溃残留。
|
|
79
|
+
* 有了它,「持锁进程已死但 PID 被系统复用」才不会让用户永久卡在「另一个进程正在准备环境」上。
|
|
80
|
+
*/
|
|
81
|
+
const STALE_LOCK_MS = 6 * 60 * 60 * 1000
|
|
82
|
+
|
|
83
|
+
async function installLock(root: string): Promise<() => Promise<void>> {
|
|
84
|
+
const path = join(root, 'environment-install.lock')
|
|
85
|
+
/** 这一把锁的归属凭据。释放时靠它判断「这份文件还是不是我的」。 */
|
|
86
|
+
const token = randomUUID()
|
|
87
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
88
|
+
try {
|
|
89
|
+
const file = await open(path, 'wx')
|
|
90
|
+
try { await file.writeFile(JSON.stringify({ token, pid: process.pid, at: new Date().toISOString() })) }
|
|
91
|
+
catch (cause) { await rm(path, { force: true }); throw cause }
|
|
92
|
+
finally { await file.close() }
|
|
93
|
+
return async () => {
|
|
94
|
+
/**
|
|
95
|
+
* 只删**自己那一把**。判据是 token,不是「路径上有没有文件」:锁可能已被另一个
|
|
96
|
+
* 进程按陈旧接管过,那时这份文件是它的 —— 删掉就等于让两个安装并发写同一个 venv。
|
|
97
|
+
*/
|
|
98
|
+
try {
|
|
99
|
+
const current = JSON.parse(await readFile(path, 'utf8')) as { token?: unknown }
|
|
100
|
+
if (current.token === token) await rm(path, { force: true })
|
|
101
|
+
} catch { /* 已被接管或已不存在:不是自己的锁就不动 */ }
|
|
102
|
+
}
|
|
103
|
+
} catch (error) {
|
|
104
|
+
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error
|
|
105
|
+
let holder: { token?: unknown; pid?: unknown; at?: unknown } | undefined
|
|
106
|
+
try { holder = JSON.parse(await readFile(path, 'utf8')) as { token?: unknown; pid?: unknown; at?: unknown } } catch { holder = undefined }
|
|
107
|
+
if (holder === undefined) {
|
|
108
|
+
/**
|
|
109
|
+
* 解析不了(0 字节、半截 JSON、磁盘满)也要能自愈。
|
|
110
|
+
*
|
|
111
|
+
* 锁是「`open('wx')` 建文件 → 再写内容」两步,中间被杀就会留下这种文件;
|
|
112
|
+
* 原来这里直接抛「安装锁异常」,于是**每一次**安装尝试都失败,只能人工删锁。
|
|
113
|
+
* 判据换成文件的 mtime:与「持锁进程还活着」同一套时限,超时就接管。
|
|
114
|
+
*/
|
|
115
|
+
const info = await stat(path).catch(() => undefined)
|
|
116
|
+
const aged = info !== undefined && Date.now() - info.mtimeMs > STALE_LOCK_MS
|
|
117
|
+
if (!aged) throw new Error('安装锁异常,请检查 resource/environment-install.lock')
|
|
118
|
+
await rm(path, { force: true })
|
|
119
|
+
continue
|
|
120
|
+
}
|
|
121
|
+
const pid = Number(holder.pid)
|
|
122
|
+
const at = Date.parse(String(holder.at ?? ''))
|
|
123
|
+
const aged = Number.isFinite(at) && Date.now() - at > STALE_LOCK_MS
|
|
124
|
+
if (!Number.isInteger(pid) || pid <= 0) {
|
|
125
|
+
// 内容不成立:没到期就如实报出来让人看一眼,到期了才当陈旧处理。
|
|
126
|
+
if (!aged) throw new Error('安装锁异常,请检查 resource/environment-install.lock')
|
|
127
|
+
} else {
|
|
128
|
+
let dead = false
|
|
129
|
+
try { process.kill(pid, 0) } catch (cause) { dead = (cause as NodeJS.ErrnoException).code === 'ESRCH' }
|
|
130
|
+
// `dead`(进程真的没了)或 `aged`(号码可能已被别人复用)任一成立都算陈旧。
|
|
131
|
+
if (!dead && !aged) throw new Error('另一个进程正在准备环境,请等待完成后重试')
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* 接管之前**再读一次**并确认还是刚才那一把。读完到删之间仍有一个极窄的窗口,
|
|
135
|
+
* 但「先删后判」会让两个进程同时接管同一把锁 —— 那才是真正的并发安装。
|
|
136
|
+
*/
|
|
137
|
+
try {
|
|
138
|
+
const again = JSON.parse(await readFile(path, 'utf8')) as { token?: unknown }
|
|
139
|
+
if (again.token !== holder.token) continue
|
|
140
|
+
} catch { continue }
|
|
141
|
+
await rm(path, { force: true })
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
throw new Error('无法获取环境安装锁')
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* 准备这个库的推理运行时。
|
|
149
|
+
*
|
|
150
|
+
* `basePython` 省略时按 `pickBasePython()` 找:`DSH_PYTHON` → DSH 自带运行时 → PATH 上的
|
|
151
|
+
* `python3`。**默认值不能再写死 `python3`**:macOS 命令行工具自带的是 3.9,而桌面端明明把
|
|
152
|
+
* 3.12 打在了自己的运行时里 —— 写死默认值会让「一键准备」在任何没配 `DSH_PYTHON` 的机器上
|
|
153
|
+
* 第一步就失败(见 `dsh-python.ts` 的注释)。
|
|
154
|
+
*/
|
|
155
|
+
export async function installRequiredEnvironment(root: string, signal: AbortSignal, stage: (label: string) => void, log: (line: string) => void, runner: typeof runProcess = runProcess, selection: ModelSelection = DEFAULT_MODEL_SELECTION, basePython?: string, hardened: (path: string) => Promise<boolean> = path => hasHardenedRuntime(path, runner)): Promise<void> {
|
|
156
|
+
await mkdir(root, { recursive: true })
|
|
157
|
+
const picked = basePython === undefined ? pickBasePython() : { path: basePython, source: 'override' as const }
|
|
158
|
+
const unlock = await installLock(root)
|
|
159
|
+
const env = {
|
|
160
|
+
...process.env,
|
|
161
|
+
UV_CACHE_DIR: join(root, 'cache', 'uv'),
|
|
162
|
+
/**
|
|
163
|
+
* uv 自己管理的解释器也落在资源目录里:它和依赖、模型一样属于「这个 Agent 的家当」,
|
|
164
|
+
* 备份 / 迁移 / 删除该一起走,而不是散到 `~/.local/share/uv/python`。
|
|
165
|
+
*/
|
|
166
|
+
UV_PYTHON_INSTALL_DIR: join(root, 'tools', 'python'),
|
|
167
|
+
HF_HOME: join(root, 'cache', 'huggingface'),
|
|
168
|
+
PIP_DISABLE_PIP_VERSION_CHECK: '1',
|
|
169
|
+
}
|
|
170
|
+
const run = (command: string, args: string[], extra: ProcessOptions = {}): Promise<string> => runner(command, args, { env, signal, timeoutMs: 30 * 60_000, onLine: log, ...extra })
|
|
171
|
+
try {
|
|
172
|
+
stage('准备模型运行时')
|
|
173
|
+
const uv = await ensureUv(root, signal, log, runner)
|
|
174
|
+
log(`基础 Python:${picked.path}(${pythonSourceLabel(picked.source)})`)
|
|
175
|
+
/**
|
|
176
|
+
* 自带运行时那份 Python 带 hardened runtime,**装不了 torch**(见 `hasHardenedRuntime`):
|
|
177
|
+
* 库校验只喂它自己 Team ID 签的 dylib,而 pip 上的原生轮子是 ad-hoc 签名的。
|
|
178
|
+
* 这时改用 uv 管的独立 CPython(不带 hardened runtime),装到 `<resource>/tools/python`。
|
|
179
|
+
*
|
|
180
|
+
* 显式给了 `DSH_PYTHON` 就不改道 —— 那是用户的选择,只把风险写进日志。
|
|
181
|
+
*/
|
|
182
|
+
const isHardened = await hardened(picked.path)
|
|
183
|
+
const managed = needsManagedInterpreter(picked, isHardened)
|
|
184
|
+
if (isHardened && picked.source === 'override') {
|
|
185
|
+
log(`注意:${picked.path} 带 hardened runtime,第三方原生扩展(torch 等)可能加载不了。`)
|
|
186
|
+
}
|
|
187
|
+
let baseVersion: string | undefined
|
|
188
|
+
if (managed) {
|
|
189
|
+
log('基础 Python:由 uv 管理的独立 CPython 3.12(自带的那份带 hardened runtime,加载不了第三方原生扩展)')
|
|
190
|
+
} else {
|
|
191
|
+
log(`基础 Python:${picked.path}(${pythonSourceLabel(picked.source)})`)
|
|
192
|
+
const versionOutput = await run(picked.path, ['-c', PYTHON_VERSION_SCRIPT], { timeoutMs: 15_000 })
|
|
193
|
+
baseVersion = versionOutput.trim().split(/\r?\n/).at(-1)
|
|
194
|
+
if (!baseVersion || !SUPPORTED_PYTHON.test(baseVersion)) {
|
|
195
|
+
/**
|
|
196
|
+
* 这句要能直接照着做。原来只说「版本不受支持」,而人看到的是自己机器上
|
|
197
|
+
* `python3 --version` 报 3.9 —— 会以为插件在挑刺,不知道桌面端本身带了 3.12。
|
|
198
|
+
*/
|
|
199
|
+
throw new Error([
|
|
200
|
+
`DSH Python 版本不受支持:${baseVersion || '无法读取'}(${picked.path},来自${pythonSourceLabel(picked.source)});需要 Python 3.10–3.13。`,
|
|
201
|
+
picked.source === 'path'
|
|
202
|
+
? '没有找到 DSH 自带的 Python(桌面端运行时里应有一份 3.12)。请安装 Python 3.10–3.13,再用环境变量 DSH_PYTHON 指向它。'
|
|
203
|
+
: `请把 DSH_PYTHON 指向一个 3.10–3.13 的解释器后重试。`,
|
|
204
|
+
].join(' '))
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
const python = managedPython(root)
|
|
208
|
+
let compatible = false
|
|
209
|
+
if (existsSync(python)) {
|
|
210
|
+
try {
|
|
211
|
+
const current = (await run(python, ['-c', PYTHON_VERSION_SCRIPT], { timeoutMs: 15_000 })).trim().split(/\r?\n/).at(-1)
|
|
212
|
+
// 管好的解释器只要求落在支持区间(它的次版本会随 uv 走),其余必须与基础解释器同一个次版本。
|
|
213
|
+
const versionOk = managed ? SUPPORTED_PYTHON.test(current ?? '') : current === baseVersion
|
|
214
|
+
/**
|
|
215
|
+
* 还要**真的**确认这个环境不带 hardened runtime。
|
|
216
|
+
*
|
|
217
|
+
* 只比版本的话,一个用自带 Python 建出来的 3.12 环境会被判成「兼容」,于是「修复」
|
|
218
|
+
* 不重建 —— 而它正是装不了 torch 的那一个。`codesign` 跟着符号链接走,
|
|
219
|
+
* 所以这里量的是它最终指向的那个解释器。
|
|
220
|
+
*/
|
|
221
|
+
compatible = versionOk && (!managed || !(await hardened(python)))
|
|
222
|
+
} catch { if (signal.aborted) throw new Error('ENVIRONMENT_CANCELLED') }
|
|
223
|
+
}
|
|
224
|
+
if (!compatible) {
|
|
225
|
+
if (existsSync(join(root, 'python'))) {
|
|
226
|
+
const backup = join(root, `python-backup-${randomUUID()}`)
|
|
227
|
+
await rename(join(root, 'python'), backup)
|
|
228
|
+
log(`原模型运行环境不可用,已保留备份:${backup}`)
|
|
229
|
+
}
|
|
230
|
+
await run(uv, ['venv',
|
|
231
|
+
...(managed ? ['--python', MANAGED_PYTHON, '--python-preference', 'only-managed'] : ['--python', picked.path, '--python-preference', 'only-system']),
|
|
232
|
+
join(root, 'python')])
|
|
233
|
+
if (managed) log(`基础 Python:${(await run(python, ['-c', PYTHON_VERSION_SCRIPT], { timeoutMs: 15_000 })).trim()}(uv 管理的独立 CPython)`)
|
|
234
|
+
}
|
|
235
|
+
stage('安装推理依赖')
|
|
236
|
+
await run(uv, ['pip', 'install', '--reinstall', '--python', python, '-r', requirementsFile])
|
|
237
|
+
const selectedModels = modelsForSelection(selection)
|
|
238
|
+
for (const model of selectedModels) {
|
|
239
|
+
stage(model.kind === 'embedding' ? '下载向量模型' : '下载重排模型')
|
|
240
|
+
await run(python, [setupScript, '--repo', model.repo, '--revision', model.revision, '--destination', join(root, 'models', model.directory)])
|
|
241
|
+
}
|
|
242
|
+
await writeModelSelection(root, selection)
|
|
243
|
+
} finally { await unlock() }
|
|
244
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { DEFAULT_MODEL_SELECTION, embeddingModel, rerankModel } from './model-catalog.js'
|
|
2
|
+
|
|
3
|
+
/** Revisions are immutable Hub commit IDs, not a moving main branch. */
|
|
4
|
+
export const MODEL_REVISIONS = {
|
|
5
|
+
'bge-small-zh-v1.5': '7999e1d3359715c523056ef9478215996d62a620',
|
|
6
|
+
'bge-base-zh-v1.5': 'f03589ceff5aac7111bd60cfc7d497ca17ecac65',
|
|
7
|
+
'bge-large-zh-v1.5': '79e7739b6ab944e86d6171e44d24c997fc1e0116',
|
|
8
|
+
'wemm-embedding-2b-quantized': '524bbe61e1b8ce4673e4e17dcf6a8694a3c45993',
|
|
9
|
+
'wemm-embedding-2b': 'bbd6cd4bf52cfc6716f752a2df80b2706720bd95',
|
|
10
|
+
'bge-reranker-base': '2cfc18c9415c912f9d8155881c133215df768a70',
|
|
11
|
+
'bge-reranker-large': '55611d7bca2a7133960a6d3b71e083071bbfc312',
|
|
12
|
+
'qwen3-reranker-0.6b': 'e61197ed45024b0ed8a2d74b80b4d909f1255473',
|
|
13
|
+
'qwen3-reranker-4b': '22e683669bc0f0bd69640a1354a6d0aebcfeede5',
|
|
14
|
+
'qwen3-reranker-8b': '77d193c791ed757ca307ee72715aa132723da912',
|
|
15
|
+
} as const
|
|
16
|
+
|
|
17
|
+
export function modelsForSelection(selection = DEFAULT_MODEL_SELECTION) {
|
|
18
|
+
const embedding = embeddingModel(selection.embedding)
|
|
19
|
+
const rerank = rerankModel(selection.rerank)
|
|
20
|
+
return [
|
|
21
|
+
{ ...embedding, kind: 'embedding' as const, revision: MODEL_REVISIONS[embedding.id] },
|
|
22
|
+
{ ...rerank, kind: 'rerank' as const, revision: MODEL_REVISIONS[rerank.id] },
|
|
23
|
+
]
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export const REQUIRED_MODELS = modelsForSelection()
|
|
27
|
+
export const UV_VERSION = '0.13.1'
|
|
28
|
+
export const ENVIRONMENT_PROBE_VERSION = 3
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Models supported by the local embedding and reranking workers. Revisions are pinned in manifest.ts.
|
|
3
|
+
*
|
|
4
|
+
* 这一份是**纯数据**:客户端也要 import 它(模型选项、下载量的文案),所以这里不能出现
|
|
5
|
+
* `node:*`。磁盘上有没有那份模型由服务端判(`modelOptionsView(root)`)。
|
|
6
|
+
*/
|
|
7
|
+
export const EMBEDDING_MODELS = [
|
|
8
|
+
{
|
|
9
|
+
id: 'bge-small-zh-v1.5', repo: 'BAAI/bge-small-zh-v1.5', directory: 'bge-small-zh-v1.5',
|
|
10
|
+
label: 'BGE Small 中文', dimension: 512, download: '约 96 MB',
|
|
11
|
+
description: '资源占用最低,适合先跑通本地知识检索。',
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
id: 'bge-base-zh-v1.5', repo: 'BAAI/bge-base-zh-v1.5', directory: 'bge-base-zh-v1.5',
|
|
15
|
+
label: 'BGE Base 中文', dimension: 768, download: '约 409 MB',
|
|
16
|
+
description: '效果与资源占用较均衡。',
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
id: 'bge-large-zh-v1.5', repo: 'BAAI/bge-large-zh-v1.5', directory: 'bge-large-zh-v1.5',
|
|
20
|
+
label: 'BGE Large 中文', dimension: 1024, download: '约 1.3 GB',
|
|
21
|
+
description: '中文向量效果优先,需要更多内存与磁盘。',
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
id: 'wemm-embedding-2b-quantized', repo: 'ewin-reg/WeMM-Embedding-2B-Quantized', directory: 'WeMM-Embedding-2B-Quantized',
|
|
25
|
+
label: 'WeMM Embedding 2B 社区量化版', dimension: 2048, download: '约 1.8 GB',
|
|
26
|
+
description: '社区维护的 INT4 / FP8 量化模型,非腾讯官方发布;模型卡标注 license: other,请查看协议。',
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
id: 'wemm-embedding-2b', repo: 'tencent/WeMM-Embedding-2B', directory: 'WeMM-Embedding-2B',
|
|
30
|
+
label: '腾讯 WeMM Embedding 2B(BF16)', dimension: 2048, download: '约 5.5 GB',
|
|
31
|
+
description: '多模态向量模型,支持文本和图片;下载与推理资源占用较高。',
|
|
32
|
+
},
|
|
33
|
+
] as const
|
|
34
|
+
|
|
35
|
+
export const RERANK_MODELS = [
|
|
36
|
+
{
|
|
37
|
+
id: 'bge-reranker-base', repo: 'BAAI/bge-reranker-base', directory: 'bge-reranker-base',
|
|
38
|
+
label: 'BGE Reranker Base', download: '约 1.1 GB',
|
|
39
|
+
description: '中文重排的轻量选项。',
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
id: 'qwen3-reranker-0.6b', repo: 'Qwen/Qwen3-Reranker-0.6B', directory: 'Qwen3-Reranker-0.6B',
|
|
43
|
+
label: 'Qwen3 Reranker 0.6B', download: '约 1.2 GB',
|
|
44
|
+
description: '支持 100 多种语言,适合先试用 Qwen 重排。',
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
id: 'bge-reranker-large', repo: 'BAAI/bge-reranker-large', directory: 'bge-reranker-large',
|
|
48
|
+
label: 'BGE Reranker Large', download: '约 2.2 GB',
|
|
49
|
+
description: '中文和英文重排效果优先,推理速度较慢。',
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
id: 'qwen3-reranker-4b', repo: 'Qwen/Qwen3-Reranker-4B', directory: 'Qwen3-Reranker-4B',
|
|
53
|
+
label: 'Qwen3 Reranker 4B', download: '约 8.1 GB',
|
|
54
|
+
description: 'Qwen 中档重排模型;请先确认磁盘与内存充足。',
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
id: 'qwen3-reranker-8b', repo: 'Qwen/Qwen3-Reranker-8B', directory: 'Qwen3-Reranker-8B',
|
|
58
|
+
label: 'Qwen3 Reranker 8B', download: '约 16.4 GB',
|
|
59
|
+
description: '资源占用最高;下载前请确认有足够磁盘与内存。',
|
|
60
|
+
},
|
|
61
|
+
] as const
|
|
62
|
+
|
|
63
|
+
export type EmbeddingModelId = (typeof EMBEDDING_MODELS)[number]['id']
|
|
64
|
+
export type RerankModelId = (typeof RERANK_MODELS)[number]['id']
|
|
65
|
+
export interface ModelSelection { embedding: EmbeddingModelId; rerank: RerankModelId }
|
|
66
|
+
export interface ModelOptionView {
|
|
67
|
+
id: string
|
|
68
|
+
label: string
|
|
69
|
+
dimension?: number
|
|
70
|
+
download: string
|
|
71
|
+
description: string
|
|
72
|
+
modelCardUrl?: string
|
|
73
|
+
/**
|
|
74
|
+
* 这份模型在资源目录里是否**完整**(有 config、有权重、有分词器词表)。
|
|
75
|
+
* 省略 = 服务端没给资源目录,不知道 —— 不知道就不显示徽标,不猜。
|
|
76
|
+
*/
|
|
77
|
+
installed?: boolean
|
|
78
|
+
}
|
|
79
|
+
export const DEFAULT_MODEL_SELECTION: ModelSelection = {
|
|
80
|
+
embedding: 'bge-small-zh-v1.5',
|
|
81
|
+
rerank: 'bge-reranker-base',
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function isModelSelection(value: unknown): value is ModelSelection {
|
|
85
|
+
if (!value || typeof value !== 'object') return false
|
|
86
|
+
const selection = value as Record<string, unknown>
|
|
87
|
+
return EMBEDDING_MODELS.some(model => model.id === selection.embedding)
|
|
88
|
+
&& RERANK_MODELS.some(model => model.id === selection.rerank)
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export function parseModelSelection(value: unknown, fallback = DEFAULT_MODEL_SELECTION): ModelSelection {
|
|
92
|
+
if (value === undefined || value === null) return { ...fallback }
|
|
93
|
+
if (!isModelSelection(value)) throw new Error('模型选择无效,请重新选择向量模型和重排模型')
|
|
94
|
+
return { embedding: value.embedding, rerank: value.rerank }
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function embeddingModel(id: EmbeddingModelId) {
|
|
98
|
+
return EMBEDDING_MODELS.find(model => model.id === id)!
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export function rerankModel(id: RerankModelId) {
|
|
102
|
+
return RERANK_MODELS.find(model => model.id === id)!
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* 把「约 1.1 GB」这样的文案换回字节数。
|
|
107
|
+
*
|
|
108
|
+
* 判据只留一份:**目录里写的就是那句文案**(顺序与大小都在那里定),再写一份数字字段迟早
|
|
109
|
+
* 会和它对不上。格式是我们自己定的(`约 <数字> MB|GB`),认不出来就返回 undefined ——
|
|
110
|
+
* 算不出总数时宁可不显示总数,也不编一个。
|
|
111
|
+
*/
|
|
112
|
+
export function downloadBytes(text: string): number | undefined {
|
|
113
|
+
const matched = /约\s*([\d.]+)\s*(MB|GB)/i.exec(text)
|
|
114
|
+
if (!matched) return undefined
|
|
115
|
+
const value = Number(matched[1])
|
|
116
|
+
if (!Number.isFinite(value)) return undefined
|
|
117
|
+
return Math.round(value * (matched[2].toUpperCase() === 'GB' ? 1024 ** 3 : 1024 ** 2))
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** 字节数 → 「约 1.2 GB / 约 96 MB」。用于把几个模型加起来报一个总数。 */
|
|
121
|
+
export function formatBytes(bytes: number): string {
|
|
122
|
+
if (bytes >= 1024 ** 3) return `约 ${(bytes / 1024 ** 3).toFixed(1)} GB`
|
|
123
|
+
return `约 ${Math.max(1, Math.round(bytes / 1024 ** 2))} MB`
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* 模型选项。
|
|
128
|
+
*
|
|
129
|
+
* `installed` 是一个**回调**而不是路径:这一份是纯数据模块(客户端也 import 它),
|
|
130
|
+
* 而「磁盘上有没有、完不完整」要用 `node:fs` 判。服务端把判据传进来,顺便也就把它
|
|
131
|
+
* 与自检那一栏的「模型结构」检查绑成了**同一个函数** —— 否则会出现「选项说已下载、
|
|
132
|
+
* 自检说文件不全」这种同一件事两种说法。
|
|
133
|
+
*
|
|
134
|
+
* 不传回调就省略 `installed`:不知道就别猜,客户端也就不显示徽标。
|
|
135
|
+
*/
|
|
136
|
+
export function modelOptionsView(installed?: (directory: string) => boolean) {
|
|
137
|
+
const present = <T extends { id: string; label: string; download: string; description: string; repo: string; directory: string }>(model: T) => ({
|
|
138
|
+
id: model.id, label: model.label, download: model.download, description: model.description,
|
|
139
|
+
modelCardUrl: `https://huggingface.co/${model.repo}`,
|
|
140
|
+
...(installed === undefined ? {} : { installed: installed(model.directory) }),
|
|
141
|
+
...('dimension' in model ? { dimension: model.dimension as number } : {}),
|
|
142
|
+
})
|
|
143
|
+
return {
|
|
144
|
+
embedding: EMBEDDING_MODELS.map(present),
|
|
145
|
+
rerank: RERANK_MODELS.map(present),
|
|
146
|
+
}
|
|
147
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process'
|
|
2
|
+
|
|
3
|
+
export interface ProcessOptions {
|
|
4
|
+
cwd?: string
|
|
5
|
+
env?: NodeJS.ProcessEnv
|
|
6
|
+
signal?: AbortSignal
|
|
7
|
+
timeoutMs?: number
|
|
8
|
+
onLine?: (line: string) => void
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/** No shell, bounded output, and only this child PID is stopped. */
|
|
12
|
+
export function runProcess(command: string, args: string[], options: ProcessOptions = {}): Promise<string> {
|
|
13
|
+
return new Promise((resolve, reject) => {
|
|
14
|
+
if (options.signal?.aborted) return reject(new Error('ENVIRONMENT_CANCELLED'))
|
|
15
|
+
const child = spawn(command, args, { cwd: options.cwd, env: options.env ?? process.env, stdio: ['ignore', 'pipe', 'pipe'] })
|
|
16
|
+
let output = '', tail = '', settled = false, stopError: Error | undefined
|
|
17
|
+
let killTimer: ReturnType<typeof setTimeout> | undefined
|
|
18
|
+
const end = (error?: Error): void => {
|
|
19
|
+
if (settled) return
|
|
20
|
+
settled = true
|
|
21
|
+
clearTimeout(timer)
|
|
22
|
+
clearTimeout(killTimer)
|
|
23
|
+
options.signal?.removeEventListener('abort', cancel)
|
|
24
|
+
if (error) reject(error)
|
|
25
|
+
else resolve(output)
|
|
26
|
+
}
|
|
27
|
+
// Keep the install lock until this owned child has actually exited.
|
|
28
|
+
const stop = (error: Error): void => {
|
|
29
|
+
if (settled || stopError) return
|
|
30
|
+
stopError = error
|
|
31
|
+
clearTimeout(timer)
|
|
32
|
+
child.kill('SIGTERM')
|
|
33
|
+
killTimer = setTimeout(() => child.kill('SIGKILL'), 1500)
|
|
34
|
+
}
|
|
35
|
+
const cancel = (): void => stop(new Error('ENVIRONMENT_CANCELLED'))
|
|
36
|
+
const timer = setTimeout(() => stop(new Error(`环境步骤超时:${command}`)), options.timeoutMs ?? 30_000)
|
|
37
|
+
options.signal?.addEventListener('abort', cancel, { once: true })
|
|
38
|
+
const receive = (data: Buffer): void => {
|
|
39
|
+
const text = data.toString()
|
|
40
|
+
output = (output + text).slice(-128_000)
|
|
41
|
+
tail = (tail + text).slice(-16_000)
|
|
42
|
+
const lines = tail.split(/[\r\n]+/)
|
|
43
|
+
tail = lines.pop() ?? ''
|
|
44
|
+
for (const line of lines) if (line.trim()) options.onLine?.(line.trim())
|
|
45
|
+
}
|
|
46
|
+
child.stdout.on('data', receive)
|
|
47
|
+
child.stderr.on('data', receive)
|
|
48
|
+
child.once('error', error => end(error))
|
|
49
|
+
child.once('close', code => {
|
|
50
|
+
if (tail.trim()) options.onLine?.(tail.trim())
|
|
51
|
+
end(stopError ?? (code === 0 ? undefined : new Error(`环境步骤失败(${code ?? '已停止'}):${output.slice(-3000).trim() || command}`)))
|
|
52
|
+
})
|
|
53
|
+
})
|
|
54
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from 'node:fs'
|
|
2
|
+
import { mkdir, rename, writeFile } from 'node:fs/promises'
|
|
3
|
+
import { join } from 'node:path'
|
|
4
|
+
import { DEFAULT_MODEL_SELECTION, embeddingModel, isModelSelection, type ModelSelection } from './model-catalog.js'
|
|
5
|
+
|
|
6
|
+
export const MODEL_SELECTION_FILE = 'environment-model-selection.json'
|
|
7
|
+
|
|
8
|
+
export function readModelSelection(resourceRoot: string): ModelSelection {
|
|
9
|
+
try {
|
|
10
|
+
const value: unknown = JSON.parse(readFileSync(join(resourceRoot, MODEL_SELECTION_FILE), 'utf8'))
|
|
11
|
+
if (isModelSelection(value)) return { embedding: value.embedding, rerank: value.rerank }
|
|
12
|
+
} catch { /* first install or damaged selection: use the shipped defaults */ }
|
|
13
|
+
return { ...DEFAULT_MODEL_SELECTION }
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function selectedEmbedding(resourceRoot: string | undefined): { path?: string; dimension?: number } {
|
|
17
|
+
if (!resourceRoot) return {}
|
|
18
|
+
const model = embeddingModel(readModelSelection(resourceRoot).embedding)
|
|
19
|
+
const path = join(resourceRoot, 'models', model.directory)
|
|
20
|
+
return existsSync(path) ? { path, dimension: model.dimension } : {}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export async function writeModelSelection(resourceRoot: string, selection: ModelSelection): Promise<void> {
|
|
24
|
+
await mkdir(resourceRoot, { recursive: true })
|
|
25
|
+
const destination = join(resourceRoot, MODEL_SELECTION_FILE)
|
|
26
|
+
const temporary = `${destination}.${process.pid}.tmp`
|
|
27
|
+
await writeFile(temporary, `${JSON.stringify(selection, null, 2)}\n`, 'utf8')
|
|
28
|
+
await rename(temporary, destination)
|
|
29
|
+
}
|