@synap-core/cli 1.10.0 → 1.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -198
- package/dist/build-info.json +6 -0
- package/dist/commands/agent-run.js +3 -2
- package/dist/commands/agent-run.js.map +1 -1
- package/dist/commands/agents.d.ts +33 -17
- package/dist/commands/agents.js +203 -237
- package/dist/commands/agents.js.map +1 -1
- package/dist/commands/app.d.ts +143 -0
- package/dist/commands/app.js +523 -0
- package/dist/commands/app.js.map +1 -0
- package/dist/commands/automate.d.ts +21 -0
- package/dist/commands/automate.js +42 -0
- package/dist/commands/automate.js.map +1 -0
- package/dist/commands/automation-schema.test.d.ts +16 -0
- package/dist/commands/automation-schema.test.js +77 -0
- package/dist/commands/automation-schema.test.js.map +1 -0
- package/dist/commands/automation.d.ts +12 -0
- package/dist/commands/automation.js +80 -35
- package/dist/commands/automation.js.map +1 -1
- package/dist/commands/bridge-setup.d.ts +14 -1
- package/dist/commands/bridge-setup.js +392 -128
- package/dist/commands/bridge-setup.js.map +1 -1
- package/dist/commands/cap-connections.d.ts +4 -4
- package/dist/commands/cap-connections.js +14 -14
- package/dist/commands/cap-connections.js.map +1 -1
- package/dist/commands/capability.d.ts +2 -0
- package/dist/commands/capability.js +297 -176
- package/dist/commands/capability.js.map +1 -1
- package/dist/commands/catalog.d.ts +275 -0
- package/dist/commands/catalog.js +481 -0
- package/dist/commands/catalog.js.map +1 -0
- package/dist/commands/cell.d.ts +155 -8
- package/dist/commands/cell.js +416 -40
- package/dist/commands/cell.js.map +1 -1
- package/dist/commands/centrality.js +3 -14
- package/dist/commands/centrality.js.map +1 -1
- package/dist/commands/connect.d.ts +38 -13
- package/dist/commands/connect.js +68 -167
- package/dist/commands/connect.js.map +1 -1
- package/dist/commands/connections.d.ts +3 -2
- package/dist/commands/connections.js +95 -25
- package/dist/commands/connections.js.map +1 -1
- package/dist/commands/context.js +9 -2
- package/dist/commands/context.js.map +1 -1
- package/dist/commands/data-extra.d.ts +34 -0
- package/dist/commands/data-extra.js +198 -25
- package/dist/commands/data-extra.js.map +1 -1
- package/dist/commands/data.d.ts +120 -2
- package/dist/commands/data.js +307 -24
- package/dist/commands/data.js.map +1 -1
- package/dist/commands/diagnose.d.ts +66 -4
- package/dist/commands/diagnose.js +329 -29
- package/dist/commands/diagnose.js.map +1 -1
- package/dist/commands/digest.js +2 -1
- package/dist/commands/digest.js.map +1 -1
- package/dist/commands/doc.d.ts +25 -4
- package/dist/commands/doc.js +100 -18
- package/dist/commands/doc.js.map +1 -1
- package/dist/commands/doctor.d.ts +130 -0
- package/dist/commands/doctor.js +814 -0
- package/dist/commands/doctor.js.map +1 -0
- package/dist/commands/events.js +3 -1
- package/dist/commands/events.js.map +1 -1
- package/dist/commands/explain.js +5 -5
- package/dist/commands/explain.js.map +1 -1
- package/dist/commands/graph.js +5 -3
- package/dist/commands/graph.js.map +1 -1
- package/dist/commands/import.d.ts +368 -13
- package/dist/commands/import.js +1096 -55
- package/dist/commands/import.js.map +1 -1
- package/dist/commands/init.d.ts +20 -9
- package/dist/commands/init.js +433 -973
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/keys.d.ts +1 -0
- package/dist/commands/keys.js +15 -2
- package/dist/commands/keys.js.map +1 -1
- package/dist/commands/knowledge.d.ts +28 -6
- package/dist/commands/knowledge.js +358 -73
- package/dist/commands/knowledge.js.map +1 -1
- package/dist/commands/launch.js +2 -2
- package/dist/commands/launch.js.map +1 -1
- package/dist/commands/lens.d.ts +37 -1
- package/dist/commands/lens.js +193 -72
- package/dist/commands/lens.js.map +1 -1
- package/dist/commands/market-authoring.d.ts +72 -0
- package/dist/commands/market-authoring.js +566 -0
- package/dist/commands/market-authoring.js.map +1 -0
- package/dist/commands/market-official.d.ts +58 -0
- package/dist/commands/market-official.js +84 -0
- package/dist/commands/market-official.js.map +1 -0
- package/dist/commands/market.d.ts +419 -20
- package/dist/commands/market.js +2124 -193
- package/dist/commands/market.js.map +1 -1
- package/dist/commands/mcp.d.ts +70 -13
- package/dist/commands/mcp.js +308 -40
- package/dist/commands/mcp.js.map +1 -1
- package/dist/commands/open.d.ts +39 -9
- package/dist/commands/open.js +80 -25
- package/dist/commands/open.js.map +1 -1
- package/dist/commands/playbook.d.ts +42 -0
- package/dist/commands/playbook.js +162 -0
- package/dist/commands/playbook.js.map +1 -0
- package/dist/commands/pod.js +1 -1
- package/dist/commands/pod.js.map +1 -1
- package/dist/commands/pods.d.ts +19 -0
- package/dist/commands/pods.js +220 -85
- package/dist/commands/pods.js.map +1 -1
- package/dist/commands/proposals-actions.d.ts +62 -0
- package/dist/commands/proposals-actions.js +146 -8
- package/dist/commands/proposals-actions.js.map +1 -1
- package/dist/commands/proposals-await.d.ts +68 -0
- package/dist/commands/proposals-await.js +248 -0
- package/dist/commands/proposals-await.js.map +1 -0
- package/dist/commands/proton-setup.d.ts +73 -0
- package/dist/commands/proton-setup.js +577 -0
- package/dist/commands/proton-setup.js.map +1 -0
- package/dist/commands/providers.d.ts +23 -0
- package/dist/commands/providers.js +126 -1
- package/dist/commands/providers.js.map +1 -1
- package/dist/commands/raycast.js +6 -2
- package/dist/commands/raycast.js.map +1 -1
- package/dist/commands/rule.d.ts +150 -0
- package/dist/commands/rule.js +447 -0
- package/dist/commands/rule.js.map +1 -0
- package/dist/commands/session-wait.d.ts +111 -0
- package/dist/commands/session-wait.js +209 -0
- package/dist/commands/session-wait.js.map +1 -0
- package/dist/commands/sessions.d.ts +52 -0
- package/dist/commands/sessions.js +479 -58
- package/dist/commands/sessions.js.map +1 -1
- package/dist/commands/skill-manage.d.ts +21 -0
- package/dist/commands/skill-manage.js +136 -19
- package/dist/commands/skill-manage.js.map +1 -1
- package/dist/commands/skills.js +19 -1
- package/dist/commands/skills.js.map +1 -1
- package/dist/commands/status.d.ts +1 -4
- package/dist/commands/status.js +5 -34
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/statusline.backoff.test.d.ts +13 -0
- package/dist/commands/statusline.backoff.test.js +100 -0
- package/dist/commands/statusline.backoff.test.js.map +1 -0
- package/dist/commands/statusline.js +16 -9
- package/dist/commands/statusline.js.map +1 -1
- package/dist/commands/sync-status.d.ts +12 -0
- package/dist/commands/sync-status.js +125 -0
- package/dist/commands/sync-status.js.map +1 -0
- package/dist/commands/templates-retire.d.ts +34 -0
- package/dist/commands/templates-retire.js +192 -0
- package/dist/commands/templates-retire.js.map +1 -0
- package/dist/commands/tools.js +6 -21
- package/dist/commands/tools.js.map +1 -1
- package/dist/commands/tracks.d.ts +51 -0
- package/dist/commands/tracks.js +291 -0
- package/dist/commands/tracks.js.map +1 -0
- package/dist/commands/update.d.ts +3 -11
- package/dist/commands/update.js +5 -134
- package/dist/commands/update.js.map +1 -1
- package/dist/commands/upload.d.ts +68 -0
- package/dist/commands/upload.js +257 -0
- package/dist/commands/upload.js.map +1 -0
- package/dist/commands/vendor.d.ts +28 -0
- package/dist/commands/vendor.js +110 -0
- package/dist/commands/vendor.js.map +1 -0
- package/dist/commands/versions.js +6 -0
- package/dist/commands/versions.js.map +1 -1
- package/dist/commands/view.js +10 -5
- package/dist/commands/view.js.map +1 -1
- package/dist/commands/whoami.d.ts +23 -3
- package/dist/commands/whoami.js +169 -23
- package/dist/commands/whoami.js.map +1 -1
- package/dist/commands/worker.d.ts +122 -0
- package/dist/commands/worker.js +435 -0
- package/dist/commands/worker.js.map +1 -0
- package/dist/commands/worker.test.d.ts +1 -0
- package/dist/commands/worker.test.js +116 -0
- package/dist/commands/worker.test.js.map +1 -0
- package/dist/commands/workspace-ops.d.ts +35 -0
- package/dist/commands/workspace-ops.js +153 -0
- package/dist/commands/workspace-ops.js.map +1 -0
- package/dist/index.d.ts +6 -6
- package/dist/index.js +1009 -171
- package/dist/index.js.map +1 -1
- package/dist/lib/agent-connections.d.ts +31 -0
- package/dist/lib/agent-connections.js +33 -0
- package/dist/lib/agent-connections.js.map +1 -0
- package/dist/lib/agent-detect.d.ts +27 -0
- package/dist/lib/agent-detect.js +71 -0
- package/dist/lib/agent-detect.js.map +1 -0
- package/dist/lib/agent-door.d.ts +138 -0
- package/dist/lib/agent-door.js +426 -0
- package/dist/lib/agent-door.js.map +1 -0
- package/dist/lib/agents-config.d.ts +2 -0
- package/dist/lib/agents-config.js +4 -9
- package/dist/lib/agents-config.js.map +1 -1
- package/dist/lib/approval-poll.d.ts +31 -2
- package/dist/lib/approval-poll.js +20 -7
- package/dist/lib/approval-poll.js.map +1 -1
- package/dist/lib/atomic-file.d.ts +48 -0
- package/dist/lib/atomic-file.js +129 -0
- package/dist/lib/atomic-file.js.map +1 -0
- package/dist/lib/auth.d.ts +14 -9
- package/dist/lib/auth.js +44 -61
- package/dist/lib/auth.js.map +1 -1
- package/dist/lib/batch-approval.d.ts +85 -0
- package/dist/lib/batch-approval.js +170 -0
- package/dist/lib/batch-approval.js.map +1 -0
- package/dist/lib/capture-lane.d.ts +6 -5
- package/dist/lib/capture-lane.js +11 -8
- package/dist/lib/capture-lane.js.map +1 -1
- package/dist/lib/capture-structure.d.ts +342 -0
- package/dist/lib/capture-structure.js +295 -0
- package/dist/lib/capture-structure.js.map +1 -0
- package/dist/lib/catalog-templates.d.ts +59 -0
- package/dist/lib/catalog-templates.js +126 -0
- package/dist/lib/catalog-templates.js.map +1 -0
- package/dist/lib/cli-build.d.ts +19 -0
- package/dist/lib/cli-build.js +37 -0
- package/dist/lib/cli-build.js.map +1 -0
- package/dist/lib/conversation-headers.d.ts +18 -0
- package/dist/lib/conversation-headers.js +32 -0
- package/dist/lib/conversation-headers.js.map +1 -0
- package/dist/lib/cp-packages.d.ts +216 -4
- package/dist/lib/cp-packages.js +414 -11
- package/dist/lib/cp-packages.js.map +1 -1
- package/dist/lib/credential-class.d.ts +117 -0
- package/dist/lib/credential-class.js +116 -0
- package/dist/lib/credential-class.js.map +1 -0
- package/dist/lib/describe-lens.d.ts +35 -0
- package/dist/lib/describe-lens.js +40 -0
- package/dist/lib/describe-lens.js.map +1 -1
- package/dist/lib/device-connect.d.ts +125 -0
- package/dist/lib/device-connect.js +256 -0
- package/dist/lib/device-connect.js.map +1 -0
- package/dist/lib/directory-lens.d.ts +99 -0
- package/dist/lib/directory-lens.js +187 -0
- package/dist/lib/directory-lens.js.map +1 -0
- package/dist/lib/exporter-coverage.d.ts +71 -0
- package/dist/lib/exporter-coverage.js +97 -0
- package/dist/lib/exporter-coverage.js.map +1 -0
- package/dist/lib/hub-client.d.ts +43 -9
- package/dist/lib/hub-client.js +429 -65
- package/dist/lib/hub-client.js.map +1 -1
- package/dist/lib/id.d.ts +23 -0
- package/dist/lib/id.js +35 -0
- package/dist/lib/id.js.map +1 -0
- package/dist/lib/init-connect.d.ts +41 -0
- package/dist/lib/init-connect.js +75 -0
- package/dist/lib/init-connect.js.map +1 -0
- package/dist/lib/init-exit.d.ts +24 -0
- package/dist/lib/init-exit.js +41 -0
- package/dist/lib/init-exit.js.map +1 -0
- package/dist/lib/init-flow.d.ts +25 -0
- package/dist/lib/init-flow.js +40 -0
- package/dist/lib/init-flow.js.map +1 -0
- package/dist/lib/init-handoff.d.ts +16 -0
- package/dist/lib/init-handoff.js +19 -0
- package/dist/lib/init-handoff.js.map +1 -0
- package/dist/lib/init-output.d.ts +18 -0
- package/dist/lib/init-output.js +43 -0
- package/dist/lib/init-output.js.map +1 -0
- package/dist/lib/init-pod.d.ts +45 -0
- package/dist/lib/init-pod.js +36 -0
- package/dist/lib/init-pod.js.map +1 -0
- package/dist/lib/installed.d.ts +210 -11
- package/dist/lib/installed.js +234 -36
- package/dist/lib/installed.js.map +1 -1
- package/dist/lib/key-approval.d.ts +57 -0
- package/dist/lib/key-approval.js +117 -0
- package/dist/lib/key-approval.js.map +1 -0
- package/dist/lib/key-diagnosis.d.ts +56 -0
- package/dist/lib/key-diagnosis.js +227 -0
- package/dist/lib/key-diagnosis.js.map +1 -0
- package/dist/lib/key-doctor.d.ts +136 -0
- package/dist/lib/key-doctor.js +286 -0
- package/dist/lib/key-doctor.js.map +1 -0
- package/dist/lib/key-locations.d.ts +120 -0
- package/dist/lib/key-locations.js +326 -0
- package/dist/lib/key-locations.js.map +1 -0
- package/dist/lib/key-source.d.ts +66 -0
- package/dist/lib/key-source.js +106 -0
- package/dist/lib/key-source.js.map +1 -0
- package/dist/lib/key-state.d.ts +70 -0
- package/dist/lib/key-state.js +115 -0
- package/dist/lib/key-state.js.map +1 -0
- package/dist/lib/kind-package.d.ts +181 -0
- package/dist/lib/kind-package.js +679 -0
- package/dist/lib/kind-package.js.map +1 -0
- package/dist/lib/knowledge-title.d.ts +24 -0
- package/dist/lib/knowledge-title.js +46 -0
- package/dist/lib/knowledge-title.js.map +1 -0
- package/dist/lib/knowledge-title.test.d.ts +1 -0
- package/dist/lib/knowledge-title.test.js +34 -0
- package/dist/lib/knowledge-title.test.js.map +1 -0
- package/dist/lib/machine-instance.d.ts +64 -0
- package/dist/lib/machine-instance.js +142 -0
- package/dist/lib/machine-instance.js.map +1 -0
- package/dist/lib/mcp-bridge.d.ts +42 -0
- package/dist/lib/mcp-bridge.js +147 -0
- package/dist/lib/mcp-bridge.js.map +1 -0
- package/dist/lib/mcp-bridge.test.d.ts +1 -0
- package/dist/lib/mcp-bridge.test.js +90 -0
- package/dist/lib/mcp-bridge.test.js.map +1 -0
- package/dist/lib/mcp-call.d.ts +30 -0
- package/dist/lib/mcp-call.js +107 -0
- package/dist/lib/mcp-call.js.map +1 -0
- package/dist/lib/mcp-snippets.d.ts +53 -0
- package/dist/lib/mcp-snippets.js +82 -0
- package/dist/lib/mcp-snippets.js.map +1 -0
- package/dist/lib/next-steps.d.ts +8 -0
- package/dist/lib/next-steps.js +31 -1
- package/dist/lib/next-steps.js.map +1 -1
- package/dist/lib/package-wire.d.ts +179 -0
- package/dist/lib/package-wire.js +68 -0
- package/dist/lib/package-wire.js.map +1 -0
- package/dist/lib/pair-link.d.ts +38 -0
- package/dist/lib/pair-link.js +68 -0
- package/dist/lib/pair-link.js.map +1 -0
- package/dist/lib/pod-wide.d.ts +40 -0
- package/dist/lib/pod-wide.js +46 -0
- package/dist/lib/pod-wide.js.map +1 -0
- package/dist/lib/pod.d.ts +119 -115
- package/dist/lib/pod.js +206 -412
- package/dist/lib/pod.js.map +1 -1
- package/dist/lib/project-ref.d.ts +9 -3
- package/dist/lib/project-ref.js +10 -4
- package/dist/lib/project-ref.js.map +1 -1
- package/dist/lib/review-credential.d.ts +119 -0
- package/dist/lib/review-credential.js +147 -0
- package/dist/lib/review-credential.js.map +1 -0
- package/dist/lib/safe-open.d.ts +49 -0
- package/dist/lib/safe-open.js +119 -0
- package/dist/lib/safe-open.js.map +1 -0
- package/dist/lib/session-lens.d.ts +40 -1
- package/dist/lib/session-lens.js +54 -4
- package/dist/lib/session-lens.js.map +1 -1
- package/dist/lib/superwhisper-import.d.ts +8 -0
- package/dist/lib/superwhisper-import.js +14 -7
- package/dist/lib/superwhisper-import.js.map +1 -1
- package/dist/lib/superwhisper-title.test.d.ts +1 -0
- package/dist/lib/superwhisper-title.test.js +18 -0
- package/dist/lib/superwhisper-title.test.js.map +1 -0
- package/dist/lib/targets.d.ts +134 -10
- package/dist/lib/targets.js +962 -258
- package/dist/lib/targets.js.map +1 -1
- package/dist/lib/template-file.d.ts +78 -0
- package/dist/lib/template-file.js +140 -0
- package/dist/lib/template-file.js.map +1 -0
- package/dist/lib/template-lifecycle.d.ts +217 -0
- package/dist/lib/template-lifecycle.js +433 -0
- package/dist/lib/template-lifecycle.js.map +1 -0
- package/dist/lib/template-steps.d.ts +88 -0
- package/dist/lib/template-steps.js +118 -0
- package/dist/lib/template-steps.js.map +1 -0
- package/dist/lib/time-ago.d.ts +2 -0
- package/dist/lib/time-ago.js +15 -0
- package/dist/lib/time-ago.js.map +1 -0
- package/dist/utils/logger.js +5 -2
- package/dist/utils/logger.js.map +1 -1
- package/package.json +25 -18
- package/skills/README.md +12 -10
- package/skills/agent-os/SKILL.md +220 -99
- package/skills/manifest.json +2 -2
- package/skills/onboard/SKILL.md +61 -25
- package/skills/synap/README.md +5 -8
- package/skills/synap/SKILL.md +1185 -301
- package/skills/synap/_order.txt +7 -0
- package/skills/synap/_skill.yaml +1 -1
- package/skills/synap/automations.md +15 -0
- package/skills/synap/capture.md +54 -151
- package/skills/synap/cli-operations.md +24 -18
- package/skills/synap/common-mistakes.md +1 -0
- package/skills/synap/concepts.md +28 -0
- package/skills/synap/content-os.md +92 -0
- package/skills/synap/creative-loop.md +15 -10
- package/skills/synap/crm.md +21 -52
- package/skills/synap/digesting-a-channel.md +25 -0
- package/skills/synap/document-embeds.md +141 -47
- package/skills/synap/escalation-ladder.md +20 -10
- package/skills/synap/focus-sessions.md +47 -21
- package/skills/synap/from-intent.md +142 -0
- package/skills/synap/governance-rules.md +4 -4
- package/skills/synap/governance.md +10 -7
- package/skills/synap/hand-off.md +33 -0
- package/skills/synap/inline-patterns.md +15 -7
- package/skills/synap/lenses.md +33 -15
- package/skills/synap/linking.md +41 -40
- package/skills/synap/mcp-remote-servers.md +64 -0
- package/skills/synap/mental-model.md +24 -22
- package/skills/synap/multi-entity-capture.md +6 -4
- package/skills/synap/operating-mode-rules.md +1 -1
- package/skills/synap/processes.md +118 -0
- package/skills/synap/quick-reference.md +5 -1
- package/skills/synap/reading.md +3 -1
- package/skills/synap/reflexes.md +12 -42
- package/skills/synap/viewframe-cells.md +38 -5
- package/skills/synap/work-flow.md +1 -1
- package/skills/synap/work-model.md +96 -0
- package/skills/synap/worked-examples.md +21 -1
- package/skills/synap/workspace-design.md +14 -10
- package/skills/synap/writes.md +165 -35
- package/skills/synap/writing-titles.md +63 -0
- package/skills/synap-app/SKILL.md +319 -0
- package/skills/synap-app/_order.txt +5 -0
- package/skills/synap-app/_skill.yaml +20 -0
- package/skills/synap-app/creating-the-base-app.md +64 -0
- package/skills/synap-app/governance.md +42 -0
- package/skills/synap-app/install-commands.md +55 -0
- package/skills/synap-app/overview.md +50 -0
- package/skills/synap-app/using-the-sdk.md +73 -0
- package/skills/synap-design/SKILL.md +474 -0
- package/skills/synap-design/_order.txt +4 -0
- package/skills/synap-design/_skill.yaml +21 -0
- package/skills/synap-design/loop.md +134 -0
- package/skills/synap-design/overview.md +44 -0
- package/skills/synap-design/principles.md +130 -0
- package/skills/synap-design/system.md +133 -0
- package/skills/synap-market/SKILL.md +625 -0
- package/skills/synap-market/_order.txt +5 -0
- package/skills/synap-market/_skill.yaml +24 -0
- package/skills/synap-market/author-and-publish.md +212 -0
- package/skills/synap-market/discover-and-fetch.md +59 -0
- package/skills/synap-market/governance-and-catalog.md +63 -0
- package/skills/synap-market/install.md +133 -0
- package/skills/synap-market/overview.md +119 -0
- package/skills/synap-schema/SKILL.md +172 -10
- package/skills/synap-schema/_order.txt +2 -0
- package/skills/synap-schema/_skill.yaml +4 -2
- package/skills/synap-schema/common-mistakes.md +4 -2
- package/skills/synap-schema/discover-profiles.md +1 -1
- package/skills/synap-schema/entity-scope.md +15 -3
- package/skills/synap-schema/extend-first.md +29 -0
- package/skills/synap-schema/extend-vs-create.md +1 -1
- package/skills/synap-schema/intent-vocabulary-ssot.md +111 -0
- package/skills/synap-schema/property-types.md +1 -1
- package/skills/synap-schema/relations.md +1 -1
- package/skills/synap-ui/SKILL.md +19 -12
- package/skills/synap-ui/_skill.yaml +1 -1
- package/skills/synap-ui/ai-companion.md +1 -1
- package/skills/synap-ui/arranging-bento.md +2 -0
- package/skills/synap-ui/bento-layouts.md +10 -10
- package/skills/synap-ui/bento-recipes.md +313 -543
- package/skills/synap-ui/four-builds.md +5 -0
- package/skills/synap-ui/view-types.md +2 -2
- package/skills/synap-ui/widget-catalog.md +22 -278
- package/templates/automations/bookmark-enrichment.automation.json +290 -13
- package/templates/automations/url-bookmark-capture.automation.json +73 -11
- package/templates/skills/authoring-playbooks/SKILL.md +6 -3
- package/templates/skills/authoring-playbooks/reference/stages-contract.md +15 -0
- package/templates/skills/authoring-playbooks/reference/worked-example.md +3 -0
- package/dist/commands/infra.d.ts +0 -21
- package/dist/commands/infra.js +0 -262
- package/dist/commands/infra.js.map +0 -1
- package/dist/lib/browser-auth.d.ts +0 -35
- package/dist/lib/browser-auth.js +0 -170
- package/dist/lib/browser-auth.js.map +0 -1
- package/dist/lib/openclaw.d.ts +0 -32
- package/dist/lib/openclaw.js +0 -164
- package/dist/lib/openclaw.js.map +0 -1
- package/dist/lib/seed.d.ts +0 -13
- package/dist/lib/seed.js +0 -135
- package/dist/lib/seed.js.map +0 -1
package/dist/commands/import.js
CHANGED
|
@@ -1,33 +1,56 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* synap import — "capture, but for files and URLs".
|
|
2
|
+
* synap import — "capture, but for files and URLs" (personal OS intake).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* POST /capture/structure
|
|
7
|
-
*
|
|
4
|
+
* Routing (INTAKE-CONSOLIDATION-PLAN Wave 1):
|
|
5
|
+
* - Superwhisper store-first → lib/superwhisper-import (unchanged)
|
|
6
|
+
* - Single file OR single URL → POST /capture/structure + /capture/execute
|
|
7
|
+
* - 2 text-like files
|
|
8
|
+
* (md / markdown / txt / csv) → ONE batch via SYNCHRONOUS REST import:
|
|
9
|
+
* POST /import/analyze → human review (or --yes / --dry-run)
|
|
10
|
+
* POST /import/apply → materialize the EXACT previewed ops
|
|
11
|
+
* - ≥CORPUS_THRESHOLD (3) such
|
|
12
|
+
* files → ONE BACKGROUND job (the sync door times out):
|
|
13
|
+
* POST /import/enqueue-corpus → { jobId }, returns immediately
|
|
14
|
+
* GET /import/corpus-job/{id} → poll to "completed"
|
|
15
|
+
* On completion the result is ONE pending governed `import.graph`
|
|
16
|
+
* proposal — NOT stored entities. `--dry-run` never takes this door
|
|
17
|
+
* (enqueueing is itself a write).
|
|
18
|
+
* - Mixed binary/URL + text → capture path for URLs/binaries;
|
|
19
|
+
* batch text via /import/* when N≥2
|
|
8
20
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
21
|
+
* Directory walks cap at DIR_FILE_CAP (sync analyze max); truncate is warned,
|
|
22
|
+
* never silent. After expansion, batch-eligible text is also capped at
|
|
23
|
+
* GLOBAL_BATCH_CAP (same ceiling) across all inputs — not only per-directory.
|
|
24
|
+
* Workspace/project from --workspace / --project (or cfg);
|
|
25
|
+
* session from --session (active session id).
|
|
11
26
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* -
|
|
15
|
-
*
|
|
27
|
+
* Optional --home-map rewrites batch item paths so the backend path heuristic
|
|
28
|
+
* (workspace **name** as a path segment) pins multi-home destinations. Example:
|
|
29
|
+
* --home-map "Projects=Builder,Posts=Content OS"
|
|
30
|
+
* rewrites `5. Projects/foo.md` → `Builder/5. Projects/foo.md`. No product
|
|
31
|
+
* defaults — user-supplied pairs only. It applies to the corpus door too: the
|
|
32
|
+
* background job forwards `items` verbatim into the SAME
|
|
33
|
+
* `ImportOrchestrator.analyzeLarge` the sync door reaches, so the path
|
|
34
|
+
* heuristic is identical on both.
|
|
16
35
|
*
|
|
17
|
-
*
|
|
18
|
-
* AI-proposed targetWorkspaceId / targetProjectId from /capture/structure win.
|
|
36
|
+
* No Company OS coupling — pure personal intake doors.
|
|
19
37
|
*/
|
|
20
38
|
import chalk from "chalk";
|
|
21
39
|
import { readFileSync, statSync, readdirSync, existsSync } from "node:fs";
|
|
22
|
-
import { extname, basename, join } from "node:path";
|
|
40
|
+
import { extname, basename, join, relative } from "node:path";
|
|
23
41
|
import * as readline from "node:readline/promises";
|
|
24
42
|
import { stdin as input, stdout as output } from "node:process";
|
|
25
43
|
import { log } from "../utils/logger.js";
|
|
26
|
-
import { resolveHubConfig, resolveUserId, hubGet, hubPost,
|
|
44
|
+
import { resolveHubConfig, resolveUserId, hubGet, hubPost, resolveActiveSessionId, renderHubError, } from "../lib/hub-client.js";
|
|
27
45
|
import { unwrapList } from "../lib/unwrapList.js";
|
|
46
|
+
import { pollForApproval } from "../lib/approval-poll.js";
|
|
47
|
+
import { FLOW, renderNextSteps } from "../lib/next-steps.js";
|
|
28
48
|
import { formatLaneLine, resolveWorkspaceName } from "../lib/capture-lane.js";
|
|
49
|
+
import { isDegraded, degradedMessage, readCaptureExecute, formatWorkspaceRouting, formatPendingWorkspaceSwitch, headlessPlacementFields, projectRoutingHints, workspaceRoutingHints, } from "../lib/capture-structure.js";
|
|
29
50
|
// ─── input classification ─────────────────────────────────────────────────────
|
|
30
51
|
const TEXT_LIKE = new Set([".md", ".markdown", ".txt", ".csv", ".json", ".html", ".htm"]);
|
|
52
|
+
/** Extensions eligible for the multi-file REST /import/analyze batch path. */
|
|
53
|
+
const BATCH_TEXT_EXTS = new Set([".md", ".markdown", ".txt", ".csv"]);
|
|
31
54
|
const MIME = {
|
|
32
55
|
".md": "text/markdown",
|
|
33
56
|
".markdown": "text/markdown",
|
|
@@ -54,26 +77,72 @@ const MIME = {
|
|
|
54
77
|
};
|
|
55
78
|
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
56
79
|
const URL_RE = /^https?:\/\//i;
|
|
57
|
-
/**
|
|
58
|
-
|
|
80
|
+
/**
|
|
81
|
+
* Cap on files pulled from a single directory — log + truncate, never silently drop.
|
|
82
|
+
* Aligned with sync /import/analyze max items (2000).
|
|
83
|
+
*/
|
|
84
|
+
const DIR_FILE_CAP = 2000;
|
|
85
|
+
/**
|
|
86
|
+
* Global cap on batch-eligible text items across ALL inputs (not only one dir).
|
|
87
|
+
* Same ceiling as sync /import/analyze max items.
|
|
88
|
+
*/
|
|
89
|
+
const GLOBAL_BATCH_CAP = 2000;
|
|
90
|
+
/** Generous timeout: deep vault structure can take minutes on large corpora. */
|
|
91
|
+
const IMPORT_ANALYZE_TIMEOUT_MS = 300_000;
|
|
92
|
+
const IMPORT_APPLY_TIMEOUT_MS = 120_000;
|
|
93
|
+
/**
|
|
94
|
+
* At or above this many batch-eligible text files, route to the BACKGROUND
|
|
95
|
+
* corpus door (`POST /import/enqueue-corpus`) instead of the synchronous
|
|
96
|
+
* `/import/analyze`.
|
|
97
|
+
*
|
|
98
|
+
* Why 3, measured rather than guessed: 3 markdown files complete in ~2m01s on
|
|
99
|
+
* the background worker, and the SAME 3 files fail 0/5 through /import/analyze,
|
|
100
|
+
* each dying at 45–48s against the pod's 45s request timeout. So 3 is the
|
|
101
|
+
* smallest N at which the synchronous door is observed to be unusable.
|
|
102
|
+
*
|
|
103
|
+
* Below it (1–2 files) the synchronous path stays: it is proven to work at that
|
|
104
|
+
* size and it is strictly more interactive — it renders the operation list, the
|
|
105
|
+
* multi-home placement and the quality report, then applies on a y/N confirm.
|
|
106
|
+
* The corpus door can offer none of that; it returns a job handle and files ONE
|
|
107
|
+
* governed proposal for later review. Routing everything through it would trade
|
|
108
|
+
* a working interactive preview for a background job on inputs that never
|
|
109
|
+
* needed one.
|
|
110
|
+
*/
|
|
111
|
+
export const CORPUS_THRESHOLD = 3;
|
|
112
|
+
/** Gap between corpus-job polls. 12 req/min — well under the hub RPM budget. */
|
|
113
|
+
const CORPUS_POLL_INTERVAL_MS = 5_000;
|
|
114
|
+
/** Heartbeat cadence for the human progress line (poll ticks are quieter). */
|
|
115
|
+
const CORPUS_PROGRESS_EVERY_MS = 30_000;
|
|
116
|
+
/** Observed worker throughput: 3 files → ~2m01s ≈ 40s/file. Rounded up. */
|
|
117
|
+
const CORPUS_MS_PER_ITEM = 45_000;
|
|
118
|
+
/** Never wait less than this, even for a 3-file corpus. */
|
|
119
|
+
const CORPUS_MIN_WAIT_MS = 10 * 60_000;
|
|
120
|
+
/**
|
|
121
|
+
* Never block a terminal longer than this. Hitting it is NOT a failure — the
|
|
122
|
+
* job keeps running server-side and the CLI prints how to re-poll.
|
|
123
|
+
*/
|
|
124
|
+
const CORPUS_MAX_WAIT_MS = 2 * 60 * 60_000;
|
|
59
125
|
function mimeFor(path) {
|
|
60
126
|
return MIME[extname(path).toLowerCase()] ?? "application/octet-stream";
|
|
61
127
|
}
|
|
62
128
|
/** Read a single file into an ImportItem (utf8 for text-like, base64 otherwise). */
|
|
63
|
-
function fileToItem(path) {
|
|
129
|
+
function fileToItem(path, displayPath) {
|
|
64
130
|
const ext = extname(path).toLowerCase();
|
|
65
131
|
const filename = basename(path);
|
|
66
132
|
const mimeType = mimeFor(path);
|
|
133
|
+
const rel = displayPath ?? path;
|
|
67
134
|
if (TEXT_LIKE.has(ext)) {
|
|
68
135
|
return {
|
|
69
136
|
kind: "file",
|
|
70
|
-
label:
|
|
137
|
+
label: rel,
|
|
138
|
+
path: rel,
|
|
71
139
|
file: { content: readFileSync(path, "utf8"), mimeType, filename, encoding: "utf8" },
|
|
72
140
|
};
|
|
73
141
|
}
|
|
74
142
|
return {
|
|
75
143
|
kind: "file",
|
|
76
|
-
label:
|
|
144
|
+
label: rel,
|
|
145
|
+
path: rel,
|
|
77
146
|
file: { content: readFileSync(path).toString("base64"), mimeType, filename, encoding: "base64" },
|
|
78
147
|
};
|
|
79
148
|
}
|
|
@@ -137,7 +206,9 @@ function expandInputs(inputs) {
|
|
|
137
206
|
truncated = true;
|
|
138
207
|
for (const f of files) {
|
|
139
208
|
try {
|
|
140
|
-
|
|
209
|
+
// Preserve vault-relative paths so wikilink/relation resolution works.
|
|
210
|
+
const rel = relative(raw, f) || basename(f);
|
|
211
|
+
items.push(fileToItem(f, rel));
|
|
141
212
|
}
|
|
142
213
|
catch (e) {
|
|
143
214
|
failures.push({ label: f, error: e.message });
|
|
@@ -155,6 +226,129 @@ function expandInputs(inputs) {
|
|
|
155
226
|
}
|
|
156
227
|
return { items, failures, truncated };
|
|
157
228
|
}
|
|
229
|
+
/** True when this item can join a multi-file /import/analyze batch. */
|
|
230
|
+
function isBatchTextItem(item) {
|
|
231
|
+
if (item.kind !== "file")
|
|
232
|
+
return false;
|
|
233
|
+
if (item.file.encoding !== "utf8")
|
|
234
|
+
return false;
|
|
235
|
+
const ext = extname(item.path).toLowerCase() || extname(item.file.filename).toLowerCase();
|
|
236
|
+
return BATCH_TEXT_EXTS.has(ext);
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Source heuristic for the batch door:
|
|
240
|
+
* - any .csv → "csv"
|
|
241
|
+
* - mostly .md/.markdown → "markdown"
|
|
242
|
+
* - default → "markdown"
|
|
243
|
+
*/
|
|
244
|
+
function sourceForBatchItems(items) {
|
|
245
|
+
if (items.some((i) => i.path.toLowerCase().endsWith(".csv")))
|
|
246
|
+
return "csv";
|
|
247
|
+
const mdCount = items.filter((i) => /\.(md|markdown)$/i.test(i.path)).length;
|
|
248
|
+
if (mdCount * 2 >= items.length)
|
|
249
|
+
return "markdown";
|
|
250
|
+
return "markdown";
|
|
251
|
+
}
|
|
252
|
+
function toBatchTextItem(item) {
|
|
253
|
+
return {
|
|
254
|
+
path: item.path,
|
|
255
|
+
content: item.file.content,
|
|
256
|
+
label: item.label,
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* Parse `--home-map "Projects=Builder,Posts=Content OS"` into pairs.
|
|
261
|
+
* Values may be workspace names or UUIDs (resolved later).
|
|
262
|
+
*/
|
|
263
|
+
export function parseHomeMap(raw) {
|
|
264
|
+
const pairs = [];
|
|
265
|
+
// Split on commas that separate pairs. Values may contain spaces but not commas.
|
|
266
|
+
for (const part of raw.split(",")) {
|
|
267
|
+
const trimmed = part.trim();
|
|
268
|
+
if (!trimmed)
|
|
269
|
+
continue;
|
|
270
|
+
const eq = trimmed.indexOf("=");
|
|
271
|
+
if (eq <= 0 || eq === trimmed.length - 1) {
|
|
272
|
+
throw new Error(`Invalid --home-map entry '${trimmed}'. Expected pathSubstring=workspaceNameOrId ` +
|
|
273
|
+
`(e.g. Projects=Builder,Posts=Content OS).`);
|
|
274
|
+
}
|
|
275
|
+
const pathSubstring = trimmed.slice(0, eq).trim();
|
|
276
|
+
const workspace = trimmed.slice(eq + 1).trim();
|
|
277
|
+
if (!pathSubstring || !workspace) {
|
|
278
|
+
throw new Error(`Invalid --home-map entry '${trimmed}'. Both pathSubstring and workspace are required.`);
|
|
279
|
+
}
|
|
280
|
+
pairs.push({ pathSubstring, workspace });
|
|
281
|
+
}
|
|
282
|
+
if (pairs.length === 0) {
|
|
283
|
+
throw new Error(`Empty --home-map. Expected pathSubstring=workspaceNameOrId pairs ` +
|
|
284
|
+
`(e.g. Projects=Builder,Posts=Content OS).`);
|
|
285
|
+
}
|
|
286
|
+
return pairs;
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* Resolve home-map values to workspace **names** (id → name when UUID given).
|
|
290
|
+
* Backend path heuristic matches segment equality against workspace names.
|
|
291
|
+
*/
|
|
292
|
+
async function resolveHomeMap(pairs, cfg) {
|
|
293
|
+
const res = (await hubGet("/workspaces", {}, cfg));
|
|
294
|
+
const list = unwrapList(res, ["workspaces"]);
|
|
295
|
+
const names = list.map((w) => String(w.name ?? w.id)).join(", ");
|
|
296
|
+
return pairs.map(({ pathSubstring, workspace }) => {
|
|
297
|
+
if (UUID_RE.test(workspace)) {
|
|
298
|
+
const match = list.find((w) => String(w.id) === workspace);
|
|
299
|
+
if (!match) {
|
|
300
|
+
throw new Error(`Home-map workspace id '${workspace}' not found.${names ? ` Available: ${names}` : ""}`);
|
|
301
|
+
}
|
|
302
|
+
const workspaceName = String(match.name ?? "").trim();
|
|
303
|
+
if (!workspaceName) {
|
|
304
|
+
throw new Error(`Home-map workspace '${workspace}' has no name — cannot rewrite paths.`);
|
|
305
|
+
}
|
|
306
|
+
return { pathSubstring, workspaceName };
|
|
307
|
+
}
|
|
308
|
+
const match = list.find((w) => String(w.name ?? "").toLowerCase() === workspace.toLowerCase());
|
|
309
|
+
if (!match) {
|
|
310
|
+
throw new Error(`Home-map workspace '${workspace}' not found.${names ? ` Available: ${names}` : ""}`);
|
|
311
|
+
}
|
|
312
|
+
return { pathSubstring, workspaceName: String(match.name) };
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* If `path` contains any map pathSubstring (case-insensitive), prefix with the
|
|
317
|
+
* resolved workspace name so deep-structure segment match pins that home.
|
|
318
|
+
* First matching pair wins. Skips rewrite when the path already starts with the
|
|
319
|
+
* workspace name segment.
|
|
320
|
+
*/
|
|
321
|
+
export function applyHomeMapToPath(path, map) {
|
|
322
|
+
if (map.length === 0 || !path)
|
|
323
|
+
return path;
|
|
324
|
+
const lower = path.toLowerCase();
|
|
325
|
+
for (const { pathSubstring, workspaceName } of map) {
|
|
326
|
+
if (!pathSubstring)
|
|
327
|
+
continue;
|
|
328
|
+
if (!lower.includes(pathSubstring.toLowerCase()))
|
|
329
|
+
continue;
|
|
330
|
+
const prefix = workspaceName + "/";
|
|
331
|
+
// Already rewritten or vault already uses workspace name as root.
|
|
332
|
+
if (path === workspaceName ||
|
|
333
|
+
path.startsWith(prefix) ||
|
|
334
|
+
path.toLowerCase().startsWith(prefix.toLowerCase())) {
|
|
335
|
+
return path;
|
|
336
|
+
}
|
|
337
|
+
return `${workspaceName}/${path}`;
|
|
338
|
+
}
|
|
339
|
+
return path;
|
|
340
|
+
}
|
|
341
|
+
/** Apply home-map path rewrite to batch text items (labels stay original). */
|
|
342
|
+
function applyHomeMapToBatchItems(items, map) {
|
|
343
|
+
if (map.length === 0)
|
|
344
|
+
return items;
|
|
345
|
+
return items.map((i) => {
|
|
346
|
+
const next = applyHomeMapToPath(i.path, map);
|
|
347
|
+
if (next === i.path)
|
|
348
|
+
return i;
|
|
349
|
+
return { ...i, path: next };
|
|
350
|
+
});
|
|
351
|
+
}
|
|
158
352
|
// ─── workspace / project name → id resolution ─────────────────────────────────
|
|
159
353
|
/** Resolve a --workspace value (id or name) to a workspace id. Throws if unknown. */
|
|
160
354
|
async function resolveWorkspaceArg(value, cfg) {
|
|
@@ -186,9 +380,6 @@ async function resolveProjectArg(value, cfg) {
|
|
|
186
380
|
// ─── rendering ─────────────────────────────────────────────────────────────────
|
|
187
381
|
function renderProposals(result) {
|
|
188
382
|
const proposals = result.proposals ?? [];
|
|
189
|
-
if (result.degraded) {
|
|
190
|
-
log.warn("AI structuring degraded — this source may not be fully extractable server-side.");
|
|
191
|
-
}
|
|
192
383
|
if (proposals.length === 0) {
|
|
193
384
|
log.dim("No entity proposals extracted.");
|
|
194
385
|
return;
|
|
@@ -221,17 +412,176 @@ function pct(c) {
|
|
|
221
412
|
}
|
|
222
413
|
/** Print the AI's proposed workspace/project routing for this item. */
|
|
223
414
|
function renderRouting(result, wsOverride, projOverride) {
|
|
224
|
-
|
|
415
|
+
// The pod's NAME for its pick, with its confidence once (never a raw id
|
|
416
|
+
// when the name is known). An override is the user's own id, shown as given.
|
|
417
|
+
const ws = wsOverride ? `${wsOverride} (override)` : formatWorkspaceRouting(result);
|
|
225
418
|
const proj = projOverride ?? result.targetProjectId ?? null;
|
|
226
|
-
const wsSrc = wsOverride ? " (override)" : pct(result.targetWorkspaceConfidence);
|
|
227
419
|
const projSrc = projOverride ? " (override)" : pct(result.targetProjectConfidence);
|
|
228
|
-
log.dim(` → workspace: ${ws ?? "pod-wide / unclear"}
|
|
420
|
+
log.dim(` → workspace: ${ws ?? "pod-wide / unclear"}`);
|
|
229
421
|
if (result.targetWorkspaceReason && !wsOverride)
|
|
230
422
|
log.dim(` ${result.targetWorkspaceReason.slice(0, 90)}`);
|
|
231
423
|
log.dim(` → project: ${proj ?? "none / unclear"}${projSrc}`);
|
|
232
424
|
if (result.targetProjectReason && !projOverride)
|
|
233
425
|
log.dim(` ${result.targetProjectReason.slice(0, 90)}`);
|
|
234
426
|
}
|
|
427
|
+
function renderBatchAnalyze(result, fileCount, source) {
|
|
428
|
+
const ops = result.operations ?? [];
|
|
429
|
+
log.heading(`▸ Batch import (${fileCount} file${fileCount !== 1 ? "s" : ""}, source=${source})`);
|
|
430
|
+
if (result.summary)
|
|
431
|
+
log.info(` ${result.summary}`);
|
|
432
|
+
log.dim(` ${ops.length} operation${ops.length !== 1 ? "s" : ""}` +
|
|
433
|
+
(result.mode ? ` · mode ${result.mode}` : "") +
|
|
434
|
+
(result.proposalId ? ` · proposal ${result.proposalId}` : "") +
|
|
435
|
+
(result.sessionId ? ` · session ${result.sessionId}` : ""));
|
|
436
|
+
if (result.stats && typeof result.stats === "object") {
|
|
437
|
+
const s = result.stats;
|
|
438
|
+
const bits = [];
|
|
439
|
+
if (s.entityCount != null)
|
|
440
|
+
bits.push(`${s.entityCount} entities`);
|
|
441
|
+
if (s.relationCount != null)
|
|
442
|
+
bits.push(`${s.relationCount} relations`);
|
|
443
|
+
if (s.itemCount != null)
|
|
444
|
+
bits.push(`${s.itemCount} items`);
|
|
445
|
+
if (s.typeCount != null)
|
|
446
|
+
bits.push(`${s.typeCount} types`);
|
|
447
|
+
if (s.itemsProcessed != null)
|
|
448
|
+
bits.push(`${s.itemsProcessed} processed`);
|
|
449
|
+
if (bits.length > 0)
|
|
450
|
+
log.dim(` stats: ${bits.join(" · ")}`);
|
|
451
|
+
}
|
|
452
|
+
// Multi-home: show where create_entity ops will land (per-op targetWorkspaceId).
|
|
453
|
+
const homes = result.homes;
|
|
454
|
+
if (homes && typeof homes === "object") {
|
|
455
|
+
const parts = [];
|
|
456
|
+
for (const [wid, n] of Object.entries(homes.byWorkspace ?? {})) {
|
|
457
|
+
parts.push(`${wid.slice(0, 8)}…×${n}`);
|
|
458
|
+
}
|
|
459
|
+
if (homes.podWide && homes.podWide > 0)
|
|
460
|
+
parts.push(`pod-wide×${homes.podWide}`);
|
|
461
|
+
if (parts.length > 0) {
|
|
462
|
+
log.info(` homes${homes.multiHome ? " (multi)" : ""}: ${parts.join(" · ")}`);
|
|
463
|
+
}
|
|
464
|
+
const proj = homes.byProject ?? {};
|
|
465
|
+
const projParts = Object.entries(proj).map(([pid, n]) => `${pid.slice(0, 8)}…×${n}`);
|
|
466
|
+
if (projParts.length > 0)
|
|
467
|
+
log.dim(` projects: ${projParts.join(" · ")}`);
|
|
468
|
+
}
|
|
469
|
+
else if (result.workspaceId) {
|
|
470
|
+
log.dim(` → workspace: ${result.workspaceId}`);
|
|
471
|
+
}
|
|
472
|
+
// Quality report — refuse / improve / apply loop
|
|
473
|
+
const q = result.quality;
|
|
474
|
+
if (q && typeof q.score === "number") {
|
|
475
|
+
log.info(` ${q.summary ?? `quality ${q.score}/100`}`);
|
|
476
|
+
const findings = q.findings ?? [];
|
|
477
|
+
for (const f of findings.slice(0, 8)) {
|
|
478
|
+
const tag = (f.severity ?? "info").toUpperCase();
|
|
479
|
+
log.dim(` [${tag}] ${f.message ?? f.id}`);
|
|
480
|
+
}
|
|
481
|
+
const ups = q.nextUpgrades ?? [];
|
|
482
|
+
if (ups.length > 0) {
|
|
483
|
+
log.info(" next upgrades:");
|
|
484
|
+
for (const u of ups.slice(0, 5))
|
|
485
|
+
log.dim(` → ${u}`);
|
|
486
|
+
}
|
|
487
|
+
if (result.proposalId) {
|
|
488
|
+
log.dim(` proposal ${result.proposalId} stores quality+ops — refuse in UI, re-import with upgrades, apply when score looks good`);
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
/**
|
|
493
|
+
* Build the outcome for an item the IS structurer could not structure.
|
|
494
|
+
*
|
|
495
|
+
* Carries `degradedReason` so `--json` reports the same cause the human path
|
|
496
|
+
* already prints via `degradedMessage()`. Without it a scripted import saw
|
|
497
|
+
* `{"status":"degraded"}` with no way to tell an auth error from an empty
|
|
498
|
+
* result — the reason existed on the wire and was dropped at this one seam.
|
|
499
|
+
*/
|
|
500
|
+
export function degradedOutcome(label, res, workspaceId, projectId) {
|
|
501
|
+
return {
|
|
502
|
+
label,
|
|
503
|
+
status: "degraded",
|
|
504
|
+
workspaceId,
|
|
505
|
+
projectId,
|
|
506
|
+
...(res.degradedReason ? { degradedReason: res.degradedReason } : {}),
|
|
507
|
+
};
|
|
508
|
+
}
|
|
509
|
+
/**
|
|
510
|
+
* The exit code for a finished import run.
|
|
511
|
+
*
|
|
512
|
+
* A degraded item produced ZERO entities — the IS structurer was down and the
|
|
513
|
+
* CLI deliberately created nothing. Exiting 0 on that told a scripted backfill
|
|
514
|
+
* "success" while most of its input was silently lost, so `degraded` is a
|
|
515
|
+
* non-zero outcome exactly like `failed`.
|
|
516
|
+
*
|
|
517
|
+
* Everything else stays 0, including a clean `--dry-run` (nothing was meant to
|
|
518
|
+
* be written), a `skipped` item (the user declined) and an `empty` one (the
|
|
519
|
+
* structurer ran fine and honestly found nothing to create).
|
|
520
|
+
*
|
|
521
|
+
* Scoped to `import` on purpose: this is `import`'s own final line, not a
|
|
522
|
+
* shared helper, so no other command's exit semantics change.
|
|
523
|
+
*/
|
|
524
|
+
export function importExitCode(outcomes) {
|
|
525
|
+
return outcomes.some((o) => o.status === "failed" || o.status === "degraded") ? 1 : 0;
|
|
526
|
+
}
|
|
527
|
+
/**
|
|
528
|
+
* The `keepRaw` half of a `/capture/execute` body — `{}` unless `--keep-raw`
|
|
529
|
+
* was passed for a FILE item.
|
|
530
|
+
*
|
|
531
|
+
* Three things it must get right, all of which the pod silently tolerates:
|
|
532
|
+
* • `content` MUST be base64. `fileToItem` reads text-like files as utf8, and
|
|
533
|
+
* the pod does `Buffer.from(content, "base64")` unconditionally — sending
|
|
534
|
+
* utf8 there stores a mangled blob with no error.
|
|
535
|
+
* • `extractedText` is echoed from the structure response so the stored
|
|
536
|
+
* document lands with a real v1 body. Skipping it leaves
|
|
537
|
+
* `document_versions.content` empty, which blinds the embedding worker, the
|
|
538
|
+
* retrieval body join and Typesense enrichment.
|
|
539
|
+
* • A URL item has no bytes to keep, so `keepRaw` is simply not sent — the pod
|
|
540
|
+
* would accept `keepRaw: true` with no file and do nothing.
|
|
541
|
+
*/
|
|
542
|
+
export function buildKeepRawPayload(opts, item, structureRes) {
|
|
543
|
+
if (!opts.keepRaw || item.kind !== "file")
|
|
544
|
+
return {};
|
|
545
|
+
const content = item.file.encoding === "base64"
|
|
546
|
+
? item.file.content
|
|
547
|
+
: Buffer.from(item.file.content, "utf8").toString("base64");
|
|
548
|
+
const extractedText = structureRes.extraction?.text;
|
|
549
|
+
return {
|
|
550
|
+
keepRaw: true,
|
|
551
|
+
file: {
|
|
552
|
+
content,
|
|
553
|
+
mimeType: item.file.mimeType,
|
|
554
|
+
filename: item.file.filename,
|
|
555
|
+
...(extractedText ? { extractedText } : {}),
|
|
556
|
+
...(structureRes.extraction?.textTruncated
|
|
557
|
+
? { extractedTextTruncated: true }
|
|
558
|
+
: {}),
|
|
559
|
+
},
|
|
560
|
+
};
|
|
561
|
+
}
|
|
562
|
+
/**
|
|
563
|
+
* One line describing what ACTUALLY happened to a kept original. The pod
|
|
564
|
+
* reports four dispositions and they are not interchangeable: a
|
|
565
|
+
* governance-parked attach is not a save, and a policy denial is not a storage
|
|
566
|
+
* failure. Returns null when nothing was kept (so nothing is claimed).
|
|
567
|
+
*/
|
|
568
|
+
export function keepRawLine(res) {
|
|
569
|
+
const sf = res.sourceFile;
|
|
570
|
+
if (!sf)
|
|
571
|
+
return null;
|
|
572
|
+
switch (sf.status) {
|
|
573
|
+
case "stored":
|
|
574
|
+
return "original kept and linked to the entity";
|
|
575
|
+
case "proposed":
|
|
576
|
+
return `original awaiting review${sf.reviewUrl ? ` — ${sf.reviewUrl}` : ""}`;
|
|
577
|
+
case "denied":
|
|
578
|
+
return `original NOT kept — policy declined it${sf.reason ? `: ${sf.reason}` : ""}`;
|
|
579
|
+
case "failed":
|
|
580
|
+
return "original NOT kept — the pod could not store it";
|
|
581
|
+
default:
|
|
582
|
+
return null;
|
|
583
|
+
}
|
|
584
|
+
}
|
|
235
585
|
async function processItem(item, opts, cfg, userId, wsOverride, projOverride, sessionId, rl) {
|
|
236
586
|
// 1. Structure — workspace hint = override ?? active (the AI may still re-route).
|
|
237
587
|
const structureBody = {
|
|
@@ -247,8 +597,20 @@ async function processItem(item, opts, cfg, userId, wsOverride, projOverride, se
|
|
|
247
597
|
// give structure a generous timeout (capture's free-text path is fast).
|
|
248
598
|
const structureRes = (await hubPost("/capture/structure", structureBody, cfg, 120_000));
|
|
249
599
|
const wsTarget = wsOverride ?? structureRes.targetWorkspaceId ?? cfg.workspaceId ?? null;
|
|
600
|
+
// What the receipt reports for the project: the user's pin, else the AI's
|
|
601
|
+
// suggestion. Only the PIN is sent as `projectId` (see the execute body).
|
|
250
602
|
const projTarget = projOverride ?? structureRes.targetProjectId ?? null;
|
|
251
603
|
const proposals = structureRes.proposals ?? [];
|
|
604
|
+
// Degraded: the IS structurer is down. Create nothing (same as `synap
|
|
605
|
+
// capture` and the browser) — report it honestly instead of materializing the
|
|
606
|
+
// raw stand-in and counting it as a successful import.
|
|
607
|
+
if (isDegraded(structureRes)) {
|
|
608
|
+
if (!opts.json) {
|
|
609
|
+
log.heading(`▸ ${item.label}`);
|
|
610
|
+
log.warn(` ${degradedMessage(structureRes)}`);
|
|
611
|
+
}
|
|
612
|
+
return degradedOutcome(item.label, structureRes, wsTarget, projTarget);
|
|
613
|
+
}
|
|
252
614
|
if (!opts.json) {
|
|
253
615
|
log.heading(`▸ ${item.label}`);
|
|
254
616
|
renderProposals(structureRes);
|
|
@@ -284,9 +646,12 @@ async function processItem(item, opts, cfg, userId, wsOverride, projOverride, se
|
|
|
284
646
|
// forward the ambient workspace + the AI's routing signal and let the
|
|
285
647
|
// backend decide (auto = AI target wins over ambient when confidence is
|
|
286
648
|
// high enough AND the user is a member).
|
|
287
|
-
//
|
|
288
|
-
//
|
|
289
|
-
//
|
|
649
|
+
// The AI's advice goes through the ONE structure → execute mapper
|
|
650
|
+
// (`captureExecuteRoutingHints`, mirrored in capture-structure.ts) as
|
|
651
|
+
// ADVISORY hints. Project: only an explicit --project is a `projectId`
|
|
652
|
+
// pin; the AI's project pick is the advisory `aiProjectId` — promoting it
|
|
653
|
+
// to `projectId` linked it outright (widening cross-workspace access)
|
|
654
|
+
// before the user ever confirmed it.
|
|
290
655
|
const executeBody = {
|
|
291
656
|
userId,
|
|
292
657
|
...(wsOverride
|
|
@@ -294,22 +659,43 @@ async function processItem(item, opts, cfg, userId, wsOverride, projOverride, se
|
|
|
294
659
|
: {
|
|
295
660
|
...(cfg.workspaceId ? { workspaceId: cfg.workspaceId } : {}),
|
|
296
661
|
workspaceRouting: "auto",
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
662
|
+
// Headless door: the structure step's PLACEMENT decides what is
|
|
663
|
+
// pinned (the ONE derivation, shared with the pod's headless doors).
|
|
664
|
+
// An AI suggestion stays a proposal (`workspaceChoice: "ignored"`).
|
|
665
|
+
...headlessPlacementFields(structureRes),
|
|
666
|
+
...workspaceRoutingHints(structureRes),
|
|
300
667
|
}),
|
|
301
|
-
...(
|
|
668
|
+
...(projOverride ? { projectId: projOverride } : projectRoutingHints(structureRes)),
|
|
302
669
|
...(sessionId ? { sessionId } : {}),
|
|
303
670
|
entities: proposals,
|
|
304
671
|
relations: structureRes.relations ?? [],
|
|
672
|
+
// --keep-raw: preserve the ORIGINAL artifact, not just what the AI read out
|
|
673
|
+
// of it. Without this the imported bytes are discarded, so the created
|
|
674
|
+
// entity has no `documentId` and no source document at all.
|
|
675
|
+
...buildKeepRawPayload(opts, item, structureRes),
|
|
305
676
|
};
|
|
306
677
|
const executeRes = (await hubPost("/capture/execute", executeBody, cfg, 60_000));
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
678
|
+
// Report what the pod ACTUALLY returned. A workspace policy can queue the
|
|
679
|
+
// write as a proposal instead of applying it — surface that honestly (real
|
|
680
|
+
// proposal/review handle) rather than hardcoding "auto"/"created".
|
|
681
|
+
const outcome = readCaptureExecute(executeRes);
|
|
682
|
+
if (outcome.proposed) {
|
|
683
|
+
if (!opts.json) {
|
|
684
|
+
log.heading(`▸ ${item.label}`);
|
|
685
|
+
log.info(` Proposed — under review, not yet stored.`);
|
|
686
|
+
if (outcome.proposalId)
|
|
687
|
+
log.dim(` proposal: ${outcome.proposalId}`);
|
|
688
|
+
if (outcome.reviewUrl)
|
|
689
|
+
log.dim(` review: ${outcome.reviewUrl}`);
|
|
690
|
+
}
|
|
691
|
+
return { label: item.label, status: "proposed", entities: 0, relations: 0, workspaceId: wsTarget, projectId: projTarget };
|
|
692
|
+
}
|
|
693
|
+
const entityCount = outcome.entitiesCreated;
|
|
694
|
+
const relCount = outcome.relationsCreated;
|
|
695
|
+
// Final resting workspace: the pre-execute target. The ladder NEVER moves a
|
|
696
|
+
// capture on an AI guess (rung 5 proposes), so there is nothing to override
|
|
697
|
+
// it with — a pending suggestion is reported separately, below.
|
|
698
|
+
const finalWorkspaceId = wsTarget;
|
|
313
699
|
if (!opts.json) {
|
|
314
700
|
const report = {
|
|
315
701
|
lane: "work",
|
|
@@ -318,11 +704,24 @@ async function processItem(item, opts, cfg, userId, wsOverride, projOverride, se
|
|
|
318
704
|
governance: "auto",
|
|
319
705
|
};
|
|
320
706
|
log.success(` ${entityCount} entit${entityCount !== 1 ? "ies" : "y"} created${relCount ? `, ${relCount} relation${relCount !== 1 ? "s" : ""}` : ""}`);
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
log.dim(` → filed into workspace ${executeRes.movedToWorkspace}`);
|
|
707
|
+
if (outcome.entitiesLinked) {
|
|
708
|
+
log.dim(` ${outcome.entitiesLinked} linked to existing entit${outcome.entitiesLinked !== 1 ? "ies" : "y"}`);
|
|
324
709
|
}
|
|
710
|
+
// Only a --project pin is filed; the AI's pick is a suggestion to confirm.
|
|
711
|
+
const projTag = projOverride
|
|
712
|
+
? ` [project ${projOverride.slice(0, 8)}]`
|
|
713
|
+
: projTarget
|
|
714
|
+
? ` [project ${projTarget.slice(0, 8)} suggested]`
|
|
715
|
+
: "";
|
|
716
|
+
console.log(" " + formatLaneLine(report) + (projTag ? chalk.dim(projTag) : ""));
|
|
717
|
+
const pending = formatPendingWorkspaceSwitch(outcome);
|
|
718
|
+
if (pending)
|
|
719
|
+
log.dim(` ${pending}`);
|
|
720
|
+
const kept = keepRawLine(executeRes);
|
|
721
|
+
if (kept)
|
|
722
|
+
log.dim(` ${kept}`);
|
|
325
723
|
}
|
|
724
|
+
const keptStatus = executeRes.sourceFile?.status;
|
|
326
725
|
return {
|
|
327
726
|
label: item.label,
|
|
328
727
|
status: "stored",
|
|
@@ -330,8 +729,541 @@ async function processItem(item, opts, cfg, userId, wsOverride, projOverride, se
|
|
|
330
729
|
relations: relCount,
|
|
331
730
|
workspaceId: finalWorkspaceId,
|
|
332
731
|
projectId: projTarget,
|
|
732
|
+
// --json parity with the human line above: a scripted import must be able
|
|
733
|
+
// to tell a kept original from a denied or failed one.
|
|
734
|
+
...(keptStatus ? { sourceFile: keptStatus } : {}),
|
|
735
|
+
};
|
|
736
|
+
}
|
|
737
|
+
/**
|
|
738
|
+
* Judge a completed job's `output`.
|
|
739
|
+
*
|
|
740
|
+
* The ONE place absence is turned into `known: false`. A pod that predates the
|
|
741
|
+
* result-carrying worker returns no output; so does a job whose handler reported
|
|
742
|
+
* nothing. Neither is evidence that every file landed, so neither may collapse
|
|
743
|
+
* into `filesFailed: 0`.
|
|
744
|
+
*/
|
|
745
|
+
export function readCorpusFileOutcome(output) {
|
|
746
|
+
if (!output || typeof output.filesFailed !== "number")
|
|
747
|
+
return { known: false };
|
|
748
|
+
return {
|
|
749
|
+
known: true,
|
|
750
|
+
filesProcessed: typeof output.filesProcessed === "number" ? output.filesProcessed : null,
|
|
751
|
+
filesFailed: output.filesFailed,
|
|
752
|
+
};
|
|
753
|
+
}
|
|
754
|
+
/**
|
|
755
|
+
* The outcome status a COMPLETED corpus job earns.
|
|
756
|
+
*
|
|
757
|
+
* The ONE decision point: files were dropped → `degraded`, which `importExitCode`
|
|
758
|
+
* already treats as non-zero (no second exit-code path). An unknown outcome stays
|
|
759
|
+
* `proposed` — the proposal genuinely exists — but the caller must print
|
|
760
|
+
* CORPUS_OUTCOME_UNAVAILABLE so the unknown is never read as "0 failed".
|
|
761
|
+
*/
|
|
762
|
+
export function corpusCompletionOutcome(output) {
|
|
763
|
+
const outcome = readCorpusFileOutcome(output);
|
|
764
|
+
if (outcome.known && outcome.filesFailed > 0) {
|
|
765
|
+
return {
|
|
766
|
+
status: "degraded",
|
|
767
|
+
degradedReason: `corpus_files_failed:${outcome.filesFailed}`,
|
|
768
|
+
};
|
|
769
|
+
}
|
|
770
|
+
return { status: "proposed" };
|
|
771
|
+
}
|
|
772
|
+
/** The message shown when the pod cannot tell us what happened per file. */
|
|
773
|
+
export const CORPUS_OUTCOME_UNAVAILABLE = "file-level outcome unavailable from this pod — it did not report how many files were structured or dropped. " +
|
|
774
|
+
"This is NOT a confirmation that every file landed: open the import.graph proposal and read quality.counts.";
|
|
775
|
+
/**
|
|
776
|
+
* Human lines for a completed corpus run's file-level outcome.
|
|
777
|
+
*
|
|
778
|
+
* Exported so the run summary, the `--job-status` re-poll and the tests all read
|
|
779
|
+
* the SAME judgement — a second copy is how "0 failed" survived on one path.
|
|
780
|
+
*/
|
|
781
|
+
export function corpusOutcomeLines(output, itemCount) {
|
|
782
|
+
const outcome = readCorpusFileOutcome(output);
|
|
783
|
+
if (!outcome.known)
|
|
784
|
+
return [CORPUS_OUTCOME_UNAVAILABLE];
|
|
785
|
+
const processed = outcome.filesProcessed ?? "?";
|
|
786
|
+
const total = typeof itemCount === "number" ? ` of ${itemCount}` : "";
|
|
787
|
+
const lines = [
|
|
788
|
+
`files: ${processed}${total} structured · ${outcome.filesFailed} failed`,
|
|
789
|
+
];
|
|
790
|
+
if (outcome.filesFailed > 0) {
|
|
791
|
+
lines.push(`${outcome.filesFailed} file(s) produced NOTHING and are absent from the proposal — re-import them alone once the cause below is addressed.`);
|
|
792
|
+
}
|
|
793
|
+
for (const f of (output?.findings ?? []).slice(0, 8)) {
|
|
794
|
+
if (!f?.message)
|
|
795
|
+
continue;
|
|
796
|
+
lines.push(`[${(f.severity ?? "info").toUpperCase()}] ${f.message}`);
|
|
797
|
+
}
|
|
798
|
+
return lines;
|
|
799
|
+
}
|
|
800
|
+
/** pg-boss states that mean "this job is over and it did not succeed". */
|
|
801
|
+
const CORPUS_DEAD_STATES = new Set(["failed", "cancelled", "expired"]);
|
|
802
|
+
/**
|
|
803
|
+
* Which batch door to use. The ONE place the threshold is applied — the human
|
|
804
|
+
* message, the --json payload and the actual POST all read this.
|
|
805
|
+
*
|
|
806
|
+
* `--dry-run` always stays on `/import/analyze`: the corpus door has no preview
|
|
807
|
+
* mode (`previewOnly` is an /import/analyze field), and enqueueing is itself a
|
|
808
|
+
* durable write. A dry run that queued a background job would be a dry run that
|
|
809
|
+
* writes — the exact thing the flag promises not to do.
|
|
810
|
+
*/
|
|
811
|
+
export function chooseBatchDoor(args) {
|
|
812
|
+
if (args.dryRun)
|
|
813
|
+
return "analyze";
|
|
814
|
+
return args.fileCount >= CORPUS_THRESHOLD ? "enqueue-corpus" : "analyze";
|
|
815
|
+
}
|
|
816
|
+
/**
|
|
817
|
+
* How long to wait for a corpus job before handing the terminal back.
|
|
818
|
+
*
|
|
819
|
+
* Scaled from measured throughput (~45s/file) so a small corpus is not waited on
|
|
820
|
+
* for an hour and a 257-file backfill is not abandoned after five minutes, then
|
|
821
|
+
* clamped: never under 10 minutes, never over 2 hours. The ceiling is safe
|
|
822
|
+
* precisely because expiry is not failure — see corpusTimeoutLines().
|
|
823
|
+
*/
|
|
824
|
+
export function corpusPollTimeoutMs(itemCount) {
|
|
825
|
+
const scaled = itemCount * CORPUS_MS_PER_ITEM;
|
|
826
|
+
return Math.min(Math.max(scaled, CORPUS_MIN_WAIT_MS), CORPUS_MAX_WAIT_MS);
|
|
827
|
+
}
|
|
828
|
+
function humanDuration(ms) {
|
|
829
|
+
const totalMin = Math.round(ms / 60_000);
|
|
830
|
+
if (totalMin < 60)
|
|
831
|
+
return `${Math.max(totalMin, 1)}m`;
|
|
832
|
+
const h = Math.floor(totalMin / 60);
|
|
833
|
+
const m = totalMin % 60;
|
|
834
|
+
return m ? `${h}h${m}m` : `${h}h`;
|
|
835
|
+
}
|
|
836
|
+
/**
|
|
837
|
+
* What we print when the wait window elapses.
|
|
838
|
+
*
|
|
839
|
+
* The job is STILL RUNNING — pg-boss holds it, the worker keeps chunking, and
|
|
840
|
+
* the governed `import.graph` proposal will still be written. Saying "failed"
|
|
841
|
+
* (or exiting non-zero) here would send a user to re-run a 257-file import that
|
|
842
|
+
* is already half-done. So: state plainly that nothing was lost, and give the
|
|
843
|
+
* exact command to re-poll.
|
|
844
|
+
*/
|
|
845
|
+
export function corpusTimeoutLines(jobId, waitedMs) {
|
|
846
|
+
return [
|
|
847
|
+
`Still running after ${humanDuration(waitedMs)} — this is not a failure.`,
|
|
848
|
+
`The job is queued on the pod and keeps going after this command exits; nothing was lost and nothing was imported twice.`,
|
|
849
|
+
`Re-check it with: synap import --job-status ${jobId}`,
|
|
850
|
+
`Or watch the review inbox — the finished import lands there as ONE import.graph proposal: synap proposals list`,
|
|
851
|
+
];
|
|
852
|
+
}
|
|
853
|
+
/** Poll a corpus job to completion. Returns the last observed status. */
|
|
854
|
+
async function pollCorpusJob(jobId, cfg, userId, itemCount, quiet) {
|
|
855
|
+
const timeoutMs = corpusPollTimeoutMs(itemCount);
|
|
856
|
+
const url = new URL(`${cfg.podUrl}/api/hub/import/corpus-job/${jobId}`);
|
|
857
|
+
url.searchParams.set("userId", userId);
|
|
858
|
+
let lastState = "created";
|
|
859
|
+
let lastProgressAt = 0;
|
|
860
|
+
const startedAt = Date.now();
|
|
861
|
+
try {
|
|
862
|
+
// REUSE the shared approval poll — same create→poll shape as CP login and
|
|
863
|
+
// agent-key approval. No second loop, no second backoff policy.
|
|
864
|
+
const status = await pollForApproval({
|
|
865
|
+
url: url.toString(),
|
|
866
|
+
headers: { Authorization: `Bearer ${cfg.apiKey}` },
|
|
867
|
+
intervalMs: CORPUS_POLL_INTERVAL_MS,
|
|
868
|
+
timeoutMs,
|
|
869
|
+
isApproved: (d) => d?.state === "completed",
|
|
870
|
+
isRejected: (d) => {
|
|
871
|
+
const state = String(d?.state ?? "");
|
|
872
|
+
// Record the terminal state here too, not only in onTick: the rejected
|
|
873
|
+
// path throws, and the catch below reports `lastState`. Reading it only
|
|
874
|
+
// from onTick would make a failed job report the state BEFORE it failed.
|
|
875
|
+
if (state)
|
|
876
|
+
lastState = state;
|
|
877
|
+
return CORPUS_DEAD_STATES.has(state);
|
|
878
|
+
},
|
|
879
|
+
onApproved: (d) => d,
|
|
880
|
+
rejectedError: "corpus import job did not complete",
|
|
881
|
+
timeoutError: "__corpus_timeout__",
|
|
882
|
+
onTick: ({ data, elapsedMs }) => {
|
|
883
|
+
const state = data?.state;
|
|
884
|
+
if (state)
|
|
885
|
+
lastState = state;
|
|
886
|
+
if (quiet)
|
|
887
|
+
return;
|
|
888
|
+
if (elapsedMs - lastProgressAt < CORPUS_PROGRESS_EVERY_MS)
|
|
889
|
+
return;
|
|
890
|
+
lastProgressAt = elapsedMs;
|
|
891
|
+
log.dim(` … ${lastState} · ${humanDuration(elapsedMs)} elapsed (waiting up to ${humanDuration(timeoutMs)})`);
|
|
892
|
+
},
|
|
893
|
+
});
|
|
894
|
+
return { status, timedOut: false, waitedMs: Date.now() - startedAt };
|
|
895
|
+
}
|
|
896
|
+
catch (e) {
|
|
897
|
+
const waitedMs = Date.now() - startedAt;
|
|
898
|
+
if (e.message === "__corpus_timeout__") {
|
|
899
|
+
return { status: { jobId, state: lastState }, timedOut: true, waitedMs };
|
|
900
|
+
}
|
|
901
|
+
// Rejected: the job reached a terminal non-success state. Report the state
|
|
902
|
+
// we actually saw rather than a generic "failed".
|
|
903
|
+
return { status: { jobId, state: lastState }, timedOut: false, waitedMs };
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
/**
|
|
907
|
+
* Enqueue a text corpus on the background door and wait for the worker.
|
|
908
|
+
*
|
|
909
|
+
* The door is ADDITIVE onto the same `ImportOrchestrator.analyzeLarge` that
|
|
910
|
+
* `/import/analyze` reaches — chunking, cross-chunk dedup and the single
|
|
911
|
+
* governed `import.graph` proposal all live server-side. This function only
|
|
912
|
+
* posts, polls, and reports.
|
|
913
|
+
*/
|
|
914
|
+
async function processCorpusImport(mappedItems, source, opts, cfg, userId, workspaceId, projOverride, sessionId) {
|
|
915
|
+
const label = `corpus:${source} (${mappedItems.length} files)`;
|
|
916
|
+
const itemCount = mappedItems.length;
|
|
917
|
+
// The enqueue payload the pod builds is {userId, workspaceId, source, items}
|
|
918
|
+
// (hub-protocol/rest/capture.ts) — projectId and sessionId are accepted by the
|
|
919
|
+
// schema and then DROPPED before the job. Warn rather than let a passed flag
|
|
920
|
+
// vanish: on this path they genuinely do not apply.
|
|
921
|
+
if (!opts.json && (projOverride || sessionId)) {
|
|
922
|
+
const dropped = [projOverride ? "--project" : null, sessionId ? "--session" : null]
|
|
923
|
+
.filter(Boolean)
|
|
924
|
+
.join(" and ");
|
|
925
|
+
log.warn(` ${dropped} ${projOverride && sessionId ? "do" : "does"} not apply to the background corpus import — ` +
|
|
926
|
+
`the pod's enqueue payload carries only userId, workspaceId, source and items. ` +
|
|
927
|
+
`Set the project/session on the resulting proposal when you review it.`);
|
|
928
|
+
}
|
|
929
|
+
if (!opts.json) {
|
|
930
|
+
log.info(`Enqueuing ${itemCount} text file${itemCount !== 1 ? "s" : ""} on the background corpus door ` +
|
|
931
|
+
`(source=${source}) — the synchronous /import/analyze door times out at this size.`);
|
|
932
|
+
}
|
|
933
|
+
const enqueueRes = (await hubPost("/import/enqueue-corpus", {
|
|
934
|
+
userId,
|
|
935
|
+
source,
|
|
936
|
+
items: mappedItems.map((i) => ({ path: i.path, content: i.content })),
|
|
937
|
+
...(workspaceId ? { workspaceId } : {}),
|
|
938
|
+
}, cfg, 60_000));
|
|
939
|
+
const jobId = enqueueRes.jobId ?? null;
|
|
940
|
+
const finalWs = enqueueRes.workspaceId ?? workspaceId ?? null;
|
|
941
|
+
const corpusBase = { jobId, itemCount, waitedMs: 0 };
|
|
942
|
+
// pg-boss can accept a send and still return no id (dedup/throttle). We were
|
|
943
|
+
// told "queued", so nothing failed — but we cannot poll a job we cannot name.
|
|
944
|
+
if (!jobId) {
|
|
945
|
+
if (!opts.json) {
|
|
946
|
+
log.warn(" The pod accepted the corpus but returned no job id — it is running, but this command cannot follow it.");
|
|
947
|
+
log.dim(" Watch for the import.graph proposal with: synap proposals list");
|
|
948
|
+
}
|
|
949
|
+
return {
|
|
950
|
+
label,
|
|
951
|
+
status: "queued",
|
|
952
|
+
workspaceId: finalWs,
|
|
953
|
+
projectId: null,
|
|
954
|
+
batch: {
|
|
955
|
+
source,
|
|
956
|
+
fileCount: itemCount,
|
|
957
|
+
proposalId: null,
|
|
958
|
+
sessionId: null,
|
|
959
|
+
corpus: { ...corpusBase, state: "unknown", timedOut: false },
|
|
960
|
+
},
|
|
961
|
+
};
|
|
962
|
+
}
|
|
963
|
+
if (!opts.json) {
|
|
964
|
+
log.dim(` job ${jobId} · polling every ${CORPUS_POLL_INTERVAL_MS / 1000}s`);
|
|
965
|
+
}
|
|
966
|
+
const { status, timedOut, waitedMs } = await pollCorpusJob(jobId, cfg, userId, itemCount, Boolean(opts.json));
|
|
967
|
+
const state = status?.state ?? "unknown";
|
|
968
|
+
const corpus = { jobId, itemCount, state, timedOut, waitedMs };
|
|
969
|
+
const batchMeta = {
|
|
970
|
+
source,
|
|
971
|
+
fileCount: itemCount,
|
|
972
|
+
proposalId: null,
|
|
973
|
+
sessionId: null,
|
|
974
|
+
corpus,
|
|
975
|
+
};
|
|
976
|
+
if (timedOut) {
|
|
977
|
+
if (!opts.json) {
|
|
978
|
+
log.blank();
|
|
979
|
+
for (const line of corpusTimeoutLines(jobId, waitedMs))
|
|
980
|
+
log.info(` ${line}`);
|
|
981
|
+
}
|
|
982
|
+
return { label, status: "queued", workspaceId: finalWs, projectId: null, batch: batchMeta };
|
|
983
|
+
}
|
|
984
|
+
if (state !== "completed") {
|
|
985
|
+
if (!opts.json) {
|
|
986
|
+
log.error(` corpus job ${jobId} ended in state "${state}" — nothing was imported.`);
|
|
987
|
+
log.hint("Check the pod's job logs, then re-run the import.");
|
|
988
|
+
}
|
|
989
|
+
return {
|
|
990
|
+
label,
|
|
991
|
+
status: "failed",
|
|
992
|
+
error: `corpus job ended in state "${state}"`,
|
|
993
|
+
workspaceId: finalWs,
|
|
994
|
+
projectId: null,
|
|
995
|
+
batch: batchMeta,
|
|
996
|
+
};
|
|
997
|
+
}
|
|
998
|
+
// Completed. The result is a PENDING governed proposal — not stored entities.
|
|
999
|
+
// Say exactly that; "created N entities" here would be a lie.
|
|
1000
|
+
//
|
|
1001
|
+
// And "completed" is NOT "every file landed": the pod structures each file
|
|
1002
|
+
// separately and records `filesFailed` on the proposal. Read that number here
|
|
1003
|
+
// or the run reports success over an unknown amount of loss.
|
|
1004
|
+
const output = status?.output ?? null;
|
|
1005
|
+
const fileOutcome = readCorpusFileOutcome(output);
|
|
1006
|
+
const lostFiles = fileOutcome.known && fileOutcome.filesFailed > 0;
|
|
1007
|
+
const corpusFull = {
|
|
1008
|
+
...corpus,
|
|
1009
|
+
outcomeKnown: fileOutcome.known,
|
|
1010
|
+
filesProcessed: fileOutcome.known ? fileOutcome.filesProcessed : null,
|
|
1011
|
+
filesFailed: fileOutcome.known ? fileOutcome.filesFailed : null,
|
|
1012
|
+
...(output?.findings ? { findings: output.findings } : {}),
|
|
1013
|
+
proposalId: output?.proposalId ?? null,
|
|
1014
|
+
};
|
|
1015
|
+
const fullBatchMeta = {
|
|
1016
|
+
...batchMeta,
|
|
1017
|
+
proposalId: output?.proposalId ?? null,
|
|
1018
|
+
corpus: corpusFull,
|
|
1019
|
+
};
|
|
1020
|
+
if (!opts.json) {
|
|
1021
|
+
const headline = ` Corpus import finished in ${humanDuration(waitedMs)}${lostFiles ? " with FILES MISSING" : ""} — ONE import.graph proposal is waiting for review.`;
|
|
1022
|
+
if (lostFiles)
|
|
1023
|
+
log.warn(headline);
|
|
1024
|
+
else
|
|
1025
|
+
log.success(headline);
|
|
1026
|
+
for (const line of corpusOutcomeLines(output, itemCount)) {
|
|
1027
|
+
if (fileOutcome.known && !lostFiles)
|
|
1028
|
+
log.dim(` ${line}`);
|
|
1029
|
+
else
|
|
1030
|
+
log.warn(` ${line}`);
|
|
1031
|
+
}
|
|
1032
|
+
log.dim(" Nothing is in the graph yet: approving the proposal is what materializes it.");
|
|
1033
|
+
renderNextSteps(FLOW.afterCorpusImport());
|
|
1034
|
+
}
|
|
1035
|
+
// Files dropped → `degraded`, the existing non-zero outcome (importExitCode);
|
|
1036
|
+
// otherwise `proposed`, the same meaning the capture path already carries.
|
|
1037
|
+
// The judgement lives in corpusCompletionOutcome so the tests exercise the
|
|
1038
|
+
// real decision rather than a copy of it.
|
|
1039
|
+
const completion = corpusCompletionOutcome(output);
|
|
1040
|
+
return {
|
|
1041
|
+
label,
|
|
1042
|
+
status: completion.status,
|
|
1043
|
+
...(completion.degradedReason
|
|
1044
|
+
? { degradedReason: completion.degradedReason }
|
|
1045
|
+
: {}),
|
|
1046
|
+
entities: 0,
|
|
1047
|
+
relations: 0,
|
|
1048
|
+
workspaceId: finalWs,
|
|
1049
|
+
projectId: null,
|
|
1050
|
+
batch: fullBatchMeta,
|
|
333
1051
|
};
|
|
334
1052
|
}
|
|
1053
|
+
// ─── multi-file batch via /import/analyze + /import/apply ─────────────────────
|
|
1054
|
+
async function processBatchImport(batchItems, opts, cfg, userId, wsOverride, projOverride, sessionId, rl, homeMap = []) {
|
|
1055
|
+
// Path rewrite for multi-home: backend matches workspace **names** in path
|
|
1056
|
+
// segments. --home-map prefixes matched paths with the resolved name.
|
|
1057
|
+
const mappedItems = applyHomeMapToBatchItems(batchItems, homeMap);
|
|
1058
|
+
const homeMapRewrites = homeMap.length > 0
|
|
1059
|
+
? mappedItems.filter((m, i) => m.path !== batchItems[i]?.path).length
|
|
1060
|
+
: 0;
|
|
1061
|
+
const source = sourceForBatchItems(mappedItems);
|
|
1062
|
+
const label = `batch:${source} (${mappedItems.length} files)`;
|
|
1063
|
+
const workspaceId = wsOverride ?? cfg.workspaceId;
|
|
1064
|
+
// Above CORPUS_THRESHOLD the synchronous door cannot finish inside the pod's
|
|
1065
|
+
// request timeout — hand the SAME mapped items (home-map already applied, caps
|
|
1066
|
+
// already enforced, source already resolved) to the background door instead.
|
|
1067
|
+
if (chooseBatchDoor({ fileCount: mappedItems.length, dryRun: opts.dryRun }) === "enqueue-corpus") {
|
|
1068
|
+
if (!opts.json && homeMapRewrites > 0) {
|
|
1069
|
+
log.dim(` home-map: rewrote ${homeMapRewrites} path${homeMapRewrites !== 1 ? "s" : ""} ` +
|
|
1070
|
+
`(workspace name as first segment for multi-home placement)`);
|
|
1071
|
+
}
|
|
1072
|
+
return processCorpusImport(mappedItems, source, opts, cfg, userId, workspaceId, projOverride, sessionId);
|
|
1073
|
+
}
|
|
1074
|
+
const analyzeBody = {
|
|
1075
|
+
userId,
|
|
1076
|
+
source,
|
|
1077
|
+
items: mappedItems.map((i) => ({ path: i.path, content: i.content })),
|
|
1078
|
+
aiStructure: true,
|
|
1079
|
+
// --dry-run must not file a durable import.graph proposal (inbox spam).
|
|
1080
|
+
// Interactive / --yes still create a real proposal for HITL apply.
|
|
1081
|
+
...(opts.dryRun ? { previewOnly: true } : {}),
|
|
1082
|
+
...(workspaceId ? { workspaceId } : {}),
|
|
1083
|
+
...(projOverride ? { projectId: projOverride } : {}),
|
|
1084
|
+
...(sessionId ? { sessionId } : {}),
|
|
1085
|
+
};
|
|
1086
|
+
if (!opts.json) {
|
|
1087
|
+
log.info(`Analyzing ${mappedItems.length} text file${mappedItems.length !== 1 ? "s" : ""} via /import/analyze (source=${source})…`);
|
|
1088
|
+
if (homeMapRewrites > 0) {
|
|
1089
|
+
log.dim(` home-map: rewrote ${homeMapRewrites} path${homeMapRewrites !== 1 ? "s" : ""} ` +
|
|
1090
|
+
`(workspace name as first segment for multi-home placement)`);
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
const analyzeRes = (await hubPost("/import/analyze", analyzeBody, cfg, IMPORT_ANALYZE_TIMEOUT_MS));
|
|
1094
|
+
const ops = analyzeRes.operations ?? [];
|
|
1095
|
+
const batchMeta = {
|
|
1096
|
+
source,
|
|
1097
|
+
fileCount: mappedItems.length,
|
|
1098
|
+
proposalId: analyzeRes.proposalId ?? null,
|
|
1099
|
+
sessionId: analyzeRes.sessionId ?? sessionId ?? null,
|
|
1100
|
+
analyze: analyzeRes,
|
|
1101
|
+
};
|
|
1102
|
+
if (!opts.json) {
|
|
1103
|
+
renderBatchAnalyze(analyzeRes, mappedItems.length, source);
|
|
1104
|
+
}
|
|
1105
|
+
if (ops.length === 0) {
|
|
1106
|
+
return {
|
|
1107
|
+
label,
|
|
1108
|
+
status: "empty",
|
|
1109
|
+
workspaceId: analyzeRes.workspaceId ?? workspaceId ?? null,
|
|
1110
|
+
projectId: projOverride ?? null,
|
|
1111
|
+
batch: batchMeta,
|
|
1112
|
+
};
|
|
1113
|
+
}
|
|
1114
|
+
// Dry-run (or --json without --yes): analyze only — no apply.
|
|
1115
|
+
if (opts.dryRun) {
|
|
1116
|
+
return {
|
|
1117
|
+
label,
|
|
1118
|
+
status: "dry-run",
|
|
1119
|
+
entities: ops.filter((o) => o.op === "create_entity").length || ops.length,
|
|
1120
|
+
relations: ops.filter((o) => o.op === "create_relation").length,
|
|
1121
|
+
workspaceId: analyzeRes.workspaceId ?? workspaceId ?? null,
|
|
1122
|
+
projectId: projOverride ?? null,
|
|
1123
|
+
batch: batchMeta,
|
|
1124
|
+
};
|
|
1125
|
+
}
|
|
1126
|
+
// Human apply default: confirm y/N unless --yes.
|
|
1127
|
+
// --json without --yes skips apply (same as per-item capture path).
|
|
1128
|
+
let confirmed = Boolean(opts.yes);
|
|
1129
|
+
if (!confirmed && !opts.json && rl) {
|
|
1130
|
+
const answer = await rl.question(chalk.bold(` Apply this import (${ops.length} operation${ops.length !== 1 ? "s" : ""})? [y/N] `));
|
|
1131
|
+
confirmed = answer.trim().toLowerCase() === "y";
|
|
1132
|
+
}
|
|
1133
|
+
if (!confirmed) {
|
|
1134
|
+
if (!opts.json)
|
|
1135
|
+
log.dim(" Skipped.");
|
|
1136
|
+
return {
|
|
1137
|
+
label,
|
|
1138
|
+
status: "skipped",
|
|
1139
|
+
workspaceId: analyzeRes.workspaceId ?? workspaceId ?? null,
|
|
1140
|
+
projectId: projOverride ?? null,
|
|
1141
|
+
batch: batchMeta,
|
|
1142
|
+
};
|
|
1143
|
+
}
|
|
1144
|
+
const applySessionId = analyzeRes.sessionId ?? sessionId;
|
|
1145
|
+
// Prefer analyze-resolved placement so apply lands where the proposal was filed.
|
|
1146
|
+
const applyWorkspaceId = analyzeRes.workspaceId ?? workspaceId ?? undefined;
|
|
1147
|
+
const applyBody = {
|
|
1148
|
+
userId,
|
|
1149
|
+
source,
|
|
1150
|
+
operations: ops,
|
|
1151
|
+
...(applyWorkspaceId ? { workspaceId: applyWorkspaceId } : {}),
|
|
1152
|
+
...(projOverride ? { projectId: projOverride } : {}),
|
|
1153
|
+
...(applySessionId ? { sessionId: applySessionId } : {}),
|
|
1154
|
+
// Client-stable idempotency namespace (U1): analyze proposalId.
|
|
1155
|
+
...(analyzeRes.proposalId
|
|
1156
|
+
? {
|
|
1157
|
+
idempotencyKey: analyzeRes.proposalId,
|
|
1158
|
+
proposalId: analyzeRes.proposalId,
|
|
1159
|
+
}
|
|
1160
|
+
: {}),
|
|
1161
|
+
};
|
|
1162
|
+
const applyRes = (await hubPost("/import/apply", applyBody, cfg, IMPORT_APPLY_TIMEOUT_MS));
|
|
1163
|
+
const created = applyRes.created ?? 0;
|
|
1164
|
+
const linked = applyRes.linked ?? 0;
|
|
1165
|
+
const finalWs = applyRes.workspaceId ?? analyzeRes.workspaceId ?? workspaceId ?? null;
|
|
1166
|
+
if (!opts.json) {
|
|
1167
|
+
log.success(` ${created} entit${created !== 1 ? "ies" : "y"} created` +
|
|
1168
|
+
(linked ? `, ${linked} linked` : ""));
|
|
1169
|
+
if (finalWs) {
|
|
1170
|
+
const report = {
|
|
1171
|
+
lane: "work",
|
|
1172
|
+
workspaceId: finalWs,
|
|
1173
|
+
workspaceName: await resolveWorkspaceName(finalWs, cfg),
|
|
1174
|
+
governance: "auto",
|
|
1175
|
+
};
|
|
1176
|
+
console.log(" " +
|
|
1177
|
+
formatLaneLine(report) +
|
|
1178
|
+
(projOverride ? chalk.dim(` [project ${projOverride.slice(0, 8)}]`) : ""));
|
|
1179
|
+
}
|
|
1180
|
+
if (applySessionId)
|
|
1181
|
+
log.dim(` session: ${applySessionId}`);
|
|
1182
|
+
}
|
|
1183
|
+
return {
|
|
1184
|
+
label,
|
|
1185
|
+
status: "stored",
|
|
1186
|
+
entities: created,
|
|
1187
|
+
relations: linked,
|
|
1188
|
+
workspaceId: finalWs,
|
|
1189
|
+
projectId: projOverride ?? null,
|
|
1190
|
+
batch: { ...batchMeta, apply: applyRes },
|
|
1191
|
+
};
|
|
1192
|
+
}
|
|
1193
|
+
// ─── `synap import --job-status <jobId>` ──────────────────────────────────────
|
|
1194
|
+
/**
|
|
1195
|
+
* Re-poll a corpus job started by an earlier `synap import`.
|
|
1196
|
+
*
|
|
1197
|
+
* This is the command `corpusTimeoutLines()` tells the user to run, so it has to
|
|
1198
|
+
* exist: a timeout message naming a command that does not resolve is worse than
|
|
1199
|
+
* no message. One GET, no waiting — it reports the current state and exits.
|
|
1200
|
+
*/
|
|
1201
|
+
export async function importJobStatus(jobId, opts) {
|
|
1202
|
+
try {
|
|
1203
|
+
if (!UUID_RE.test(jobId)) {
|
|
1204
|
+
console.error(chalk.red(`'${jobId}' is not a job id — pass the UUID printed when the import was enqueued.`));
|
|
1205
|
+
process.exit(1);
|
|
1206
|
+
}
|
|
1207
|
+
const cfg = await resolveHubConfig(opts);
|
|
1208
|
+
const userId = await resolveUserId(cfg);
|
|
1209
|
+
const res = (await hubGet(`/import/corpus-job/${jobId}`, { userId }, cfg));
|
|
1210
|
+
const state = res.state ?? "unknown";
|
|
1211
|
+
const output = res.output ?? null;
|
|
1212
|
+
const fileOutcome = readCorpusFileOutcome(output);
|
|
1213
|
+
const lostFiles = fileOutcome.known && fileOutcome.filesFailed > 0;
|
|
1214
|
+
if (opts.json) {
|
|
1215
|
+
console.log(JSON.stringify({
|
|
1216
|
+
jobId,
|
|
1217
|
+
state,
|
|
1218
|
+
createdOn: res.createdOn ?? null,
|
|
1219
|
+
completedOn: res.completedOn ?? null,
|
|
1220
|
+
// Same UNKNOWN-vs-zero distinction the run path makes.
|
|
1221
|
+
outcomeKnown: state === "completed" ? fileOutcome.known : false,
|
|
1222
|
+
filesProcessed: fileOutcome.known ? fileOutcome.filesProcessed : null,
|
|
1223
|
+
filesFailed: fileOutcome.known ? fileOutcome.filesFailed : null,
|
|
1224
|
+
...(output?.findings ? { findings: output.findings } : {}),
|
|
1225
|
+
proposalId: output?.proposalId ?? null,
|
|
1226
|
+
}, null, 2));
|
|
1227
|
+
if (state === "completed" && lostFiles)
|
|
1228
|
+
process.exit(1);
|
|
1229
|
+
return;
|
|
1230
|
+
}
|
|
1231
|
+
log.heading(`Corpus import job ${jobId}`);
|
|
1232
|
+
log.info(` state: ${state}`);
|
|
1233
|
+
if (res.createdOn)
|
|
1234
|
+
log.dim(` created: ${res.createdOn}`);
|
|
1235
|
+
if (res.completedOn)
|
|
1236
|
+
log.dim(` completed: ${res.completedOn}`);
|
|
1237
|
+
if (state === "completed") {
|
|
1238
|
+
if (lostFiles) {
|
|
1239
|
+
log.warn(" Finished with FILES MISSING — ONE import.graph proposal is waiting for review.");
|
|
1240
|
+
}
|
|
1241
|
+
else {
|
|
1242
|
+
log.success(" Finished — ONE import.graph proposal is waiting for review.");
|
|
1243
|
+
}
|
|
1244
|
+
for (const line of corpusOutcomeLines(output)) {
|
|
1245
|
+
if (fileOutcome.known && !lostFiles)
|
|
1246
|
+
log.dim(` ${line}`);
|
|
1247
|
+
else
|
|
1248
|
+
log.warn(` ${line}`);
|
|
1249
|
+
}
|
|
1250
|
+
renderNextSteps(FLOW.afterCorpusImport());
|
|
1251
|
+
// A run that lost files is not a success on the re-poll path either.
|
|
1252
|
+
if (lostFiles)
|
|
1253
|
+
process.exit(1);
|
|
1254
|
+
return;
|
|
1255
|
+
}
|
|
1256
|
+
if (CORPUS_DEAD_STATES.has(state)) {
|
|
1257
|
+
log.error(` The job ended in state "${state}" — nothing was imported.`);
|
|
1258
|
+
process.exit(1);
|
|
1259
|
+
}
|
|
1260
|
+
log.info(" Still running — nothing was lost. Re-run this command later.");
|
|
1261
|
+
}
|
|
1262
|
+
catch (e) {
|
|
1263
|
+
renderHubError(e);
|
|
1264
|
+
process.exit(1);
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
335
1267
|
// ─── command entrypoint ─────────────────────────────────────────────────────────
|
|
336
1268
|
export async function importData(inputs, opts) {
|
|
337
1269
|
try {
|
|
@@ -367,10 +1299,17 @@ export async function importData(inputs, opts) {
|
|
|
367
1299
|
// Resolve routing overrides (id or name) up front so a bad value fails fast.
|
|
368
1300
|
const wsOverride = opts.workspace ? await resolveWorkspaceArg(opts.workspace, cfg) : undefined;
|
|
369
1301
|
const projOverride = opts.project ? await resolveProjectArg(opts.project, cfg) : undefined;
|
|
370
|
-
const sessionId = opts.session ?
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
1302
|
+
const sessionId = opts.session ? resolveActiveSessionId(cfg.podUrl) : undefined;
|
|
1303
|
+
// Optional multi-home path map (no product defaults — user pairs only).
|
|
1304
|
+
let homeMap = [];
|
|
1305
|
+
if (opts.homeMap) {
|
|
1306
|
+
homeMap = await resolveHomeMap(parseHomeMap(opts.homeMap), cfg);
|
|
1307
|
+
}
|
|
1308
|
+
const { items, failures, truncated: dirTruncated } = expandInputs(inputs);
|
|
1309
|
+
let truncated = dirTruncated;
|
|
1310
|
+
if (dirTruncated && !opts.json) {
|
|
1311
|
+
log.warn(`A directory exceeded ${DIR_FILE_CAP} files — only the first ${DIR_FILE_CAP} were taken ` +
|
|
1312
|
+
`(sync /import/analyze max). Narrow the path, split the import, or use a smaller folder.`);
|
|
374
1313
|
}
|
|
375
1314
|
if (items.length === 0) {
|
|
376
1315
|
if (opts.json) {
|
|
@@ -383,15 +1322,74 @@ export async function importData(inputs, opts) {
|
|
|
383
1322
|
}
|
|
384
1323
|
process.exit(failures.length > 0 ? 1 : 0);
|
|
385
1324
|
}
|
|
1325
|
+
// Partition: N≥2 batch-eligible text → /import/*; rest → per-item capture.
|
|
1326
|
+
// Single text file stays on capture structure/execute (same as a lone URL).
|
|
1327
|
+
const batchFileItems = [];
|
|
1328
|
+
const captureItems = [];
|
|
1329
|
+
// `--keep-raw` is a PER-FILE disposition and the batch door
|
|
1330
|
+
// (/import/analyze → /import/apply) has no `keepRaw` half at all. Routing a
|
|
1331
|
+
// kept-raw text file through it would accept the flag and silently discard
|
|
1332
|
+
// every original — an inert flag rendered as a live one. Send those items
|
|
1333
|
+
// down the per-item capture path instead, which is where `keepRaw` exists.
|
|
1334
|
+
for (const item of items) {
|
|
1335
|
+
if (isBatchTextItem(item) && !opts.keepRaw) {
|
|
1336
|
+
batchFileItems.push(item);
|
|
1337
|
+
}
|
|
1338
|
+
else {
|
|
1339
|
+
captureItems.push(item);
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
if (opts.keepRaw && !opts.json && captureItems.length > 1) {
|
|
1343
|
+
log.dim(`--keep-raw: importing ${captureItems.length} items one by one (the batch door cannot keep originals).`);
|
|
1344
|
+
}
|
|
1345
|
+
// Global batch cap: multi-dir / multi-file inputs can exceed the sync max
|
|
1346
|
+
// even when no single directory hit DIR_FILE_CAP.
|
|
1347
|
+
if (batchFileItems.length > GLOBAL_BATCH_CAP) {
|
|
1348
|
+
truncated = true;
|
|
1349
|
+
if (!opts.json) {
|
|
1350
|
+
log.warn(`Batch has ${batchFileItems.length} text files — truncating to ${GLOBAL_BATCH_CAP} ` +
|
|
1351
|
+
`(sync /import/analyze max). Split the import or narrow the paths.`);
|
|
1352
|
+
}
|
|
1353
|
+
batchFileItems.length = GLOBAL_BATCH_CAP;
|
|
1354
|
+
}
|
|
1355
|
+
const useBatch = batchFileItems.length >= 2;
|
|
1356
|
+
const batchCandidates = useBatch
|
|
1357
|
+
? batchFileItems.map(toBatchTextItem)
|
|
1358
|
+
: [];
|
|
1359
|
+
if (!useBatch) {
|
|
1360
|
+
captureItems.push(...batchFileItems);
|
|
1361
|
+
}
|
|
386
1362
|
if (!opts.json) {
|
|
387
|
-
|
|
1363
|
+
const parts = [];
|
|
1364
|
+
if (useBatch)
|
|
1365
|
+
parts.push(`${batchCandidates.length} text files (batch import)`);
|
|
1366
|
+
if (captureItems.length > 0) {
|
|
1367
|
+
parts.push(`${captureItems.length} item${captureItems.length !== 1 ? "s" : ""} (capture pipeline)`);
|
|
1368
|
+
}
|
|
1369
|
+
log.info(`Importing ${parts.join(" + ") || `${items.length} item${items.length !== 1 ? "s" : ""}`}` +
|
|
1370
|
+
`${opts.dryRun ? " (dry run)" : ""}…`);
|
|
388
1371
|
}
|
|
389
1372
|
const outcomes = [];
|
|
390
1373
|
// One shared readline for all interactive confirmations (human, non-json).
|
|
391
1374
|
const interactive = !opts.json && !opts.dryRun && !opts.yes;
|
|
392
1375
|
const rl = interactive ? readline.createInterface({ input, output }) : undefined;
|
|
393
1376
|
try {
|
|
394
|
-
|
|
1377
|
+
if (useBatch) {
|
|
1378
|
+
try {
|
|
1379
|
+
outcomes.push(await processBatchImport(batchCandidates, opts, cfg, userId, wsOverride, projOverride, sessionId, rl, homeMap));
|
|
1380
|
+
}
|
|
1381
|
+
catch (e) {
|
|
1382
|
+
const error = e.message;
|
|
1383
|
+
if (!opts.json)
|
|
1384
|
+
log.error(` batch import: ${error}`);
|
|
1385
|
+
outcomes.push({
|
|
1386
|
+
label: `batch (${batchCandidates.length} files)`,
|
|
1387
|
+
status: "failed",
|
|
1388
|
+
error,
|
|
1389
|
+
});
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
for (const item of captureItems) {
|
|
395
1393
|
try {
|
|
396
1394
|
outcomes.push(await processItem(item, opts, cfg, userId, wsOverride, projOverride, sessionId, rl));
|
|
397
1395
|
}
|
|
@@ -411,7 +1409,19 @@ export async function importData(inputs, opts) {
|
|
|
411
1409
|
for (const f of failures)
|
|
412
1410
|
outcomes.push({ label: f.label, status: "failed", error: f.error });
|
|
413
1411
|
if (opts.json) {
|
|
414
|
-
|
|
1412
|
+
// Surface analyze/apply payloads for the batch path (agents need ops + receipt).
|
|
1413
|
+
const batchOutcome = outcomes.find((o) => o.batch);
|
|
1414
|
+
console.log(JSON.stringify({
|
|
1415
|
+
imported: outcomes,
|
|
1416
|
+
truncated,
|
|
1417
|
+
...(batchOutcome?.batch?.analyze ? { analyze: batchOutcome.batch.analyze } : {}),
|
|
1418
|
+
...(batchOutcome?.batch?.apply ? { apply: batchOutcome.batch.apply } : {}),
|
|
1419
|
+
// Background corpus path: a script needs the jobId + final state to
|
|
1420
|
+
// decide whether to re-poll, and nextSteps to chain the review.
|
|
1421
|
+
...(batchOutcome?.batch?.corpus
|
|
1422
|
+
? { corpus: batchOutcome.batch.corpus, nextSteps: FLOW.afterCorpusImport() }
|
|
1423
|
+
: {}),
|
|
1424
|
+
}, null, 2));
|
|
415
1425
|
}
|
|
416
1426
|
else {
|
|
417
1427
|
const stored = outcomes.filter((o) => o.status === "stored");
|
|
@@ -419,18 +1429,49 @@ export async function importData(inputs, opts) {
|
|
|
419
1429
|
const failed = outcomes.filter((o) => o.status === "failed");
|
|
420
1430
|
log.blank();
|
|
421
1431
|
log.heading("Import summary");
|
|
422
|
-
|
|
1432
|
+
const proposed = outcomes.filter((o) => o.status === "proposed").length;
|
|
1433
|
+
const degraded = outcomes.filter((o) => o.status === "degraded").length;
|
|
1434
|
+
const queued = outcomes.filter((o) => o.status === "queued").length;
|
|
1435
|
+
// Background corpus path: ONE outcome represents ONE JOB covering N files.
|
|
1436
|
+
// So this line's counters are denominated in JOBS, and the file-level truth
|
|
1437
|
+
// is a separate line below.
|
|
1438
|
+
const corpusMeta = outcomes.find((o) => o.batch?.corpus)?.batch?.corpus;
|
|
1439
|
+
const corpusLost = (corpusMeta?.filesFailed ?? 0) > 0;
|
|
1440
|
+
const noun = corpusMeta ? "job" : "item";
|
|
1441
|
+
log.info(`${outcomes.length} ${noun}${outcomes.length !== 1 ? "s" : ""} · ` +
|
|
423
1442
|
`${stored.length} stored (${totalEntities} entit${totalEntities !== 1 ? "ies" : "y"}) · ` +
|
|
1443
|
+
(proposed ? `${proposed} proposed · ` : "") +
|
|
1444
|
+
(queued ? `${queued} still running · ` : "") +
|
|
424
1445
|
`${outcomes.filter((o) => o.status === "dry-run").length} dry-run · ` +
|
|
425
1446
|
`${outcomes.filter((o) => o.status === "skipped").length} skipped · ` +
|
|
426
1447
|
`${outcomes.filter((o) => o.status === "empty").length} empty · ` +
|
|
427
|
-
`${
|
|
1448
|
+
(degraded ? `${degraded} degraded · ` : "") +
|
|
1449
|
+
// A corpus run that LOST files expresses that loss one line below, in
|
|
1450
|
+
// files. Printing "0 failed" here too — same word, different
|
|
1451
|
+
// denominator, three lines apart — restates the exact sentence this
|
|
1452
|
+
// whole wave existed to delete. Job-level failure is already carried
|
|
1453
|
+
// by `degraded`, so the counter is dropped rather than qualified.
|
|
1454
|
+
(corpusLost ? "" : `${failed.length} failed`));
|
|
428
1455
|
for (const f of failed)
|
|
429
1456
|
log.dim(` ✗ ${f.label}: ${f.error ?? "unknown error"}`);
|
|
1457
|
+
if (corpusMeta && !corpusMeta.timedOut) {
|
|
1458
|
+
if (corpusMeta.outcomeKnown) {
|
|
1459
|
+
const line = `files: ${corpusMeta.filesProcessed ?? "?"} of ${corpusMeta.itemCount} structured · ${corpusMeta.filesFailed ?? 0} failed`;
|
|
1460
|
+
if (corpusLost)
|
|
1461
|
+
log.warn(` ${line}`);
|
|
1462
|
+
else
|
|
1463
|
+
log.info(` ${line}`);
|
|
1464
|
+
}
|
|
1465
|
+
else if (corpusMeta.state === "completed") {
|
|
1466
|
+
log.warn(` ${CORPUS_OUTCOME_UNAVAILABLE}`);
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
430
1469
|
}
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
1470
|
+
// Only hard-exit on a non-zero code: a plain `return` lets Node flush the
|
|
1471
|
+
// (possibly large) --json payload it just wrote to stdout.
|
|
1472
|
+
const code = importExitCode(outcomes);
|
|
1473
|
+
if (code !== 0)
|
|
1474
|
+
process.exit(code);
|
|
434
1475
|
}
|
|
435
1476
|
catch (e) {
|
|
436
1477
|
renderHubError(e);
|