@gajae-code/coding-agent 0.17.2 → 0.17.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +267 -0
- package/dist/types/cli/daemon-cli.d.ts +3 -1
- package/dist/types/cli/github-release.d.ts +17 -0
- package/dist/types/cli/public-command-entry.d.ts +56 -0
- package/dist/types/cli/public-command-errors.d.ts +126 -0
- package/dist/types/cli/public-command-evidence.d.ts +99 -0
- package/dist/types/cli/public-command-help.d.ts +65 -0
- package/dist/types/cli/public-command-registry.d.ts +69 -0
- package/dist/types/cli/session-picker.d.ts +1 -1
- package/dist/types/cli/setup-cli.d.ts +1 -0
- package/dist/types/cli/skills-cli.d.ts +5 -0
- package/dist/types/cli.d.ts +3 -0
- package/dist/types/commands/daemon.d.ts +4 -50
- package/dist/types/commands/sdk.d.ts +20 -37
- package/dist/types/commands/setup.d.ts +3 -0
- package/dist/types/commands/skills.d.ts +9 -0
- package/dist/types/config/autorouting-tier-map.d.ts +1049 -3
- package/dist/types/config/file-lock.d.ts +13 -3
- package/dist/types/config/model-profile-activation.d.ts +45 -1
- package/dist/types/config/model-registry.d.ts +14 -3
- package/dist/types/config/model-resolver.d.ts +1 -1
- package/dist/types/config/settings-schema.d.ts +11 -0
- package/dist/types/coordinator-mcp/question-state.d.ts +6 -1
- package/dist/types/daemon/operator-contract.d.ts +5 -1
- package/dist/types/eval/js/worker-protocol.d.ts +2 -0
- package/dist/types/exec/bash-executor.d.ts +2 -0
- package/dist/types/extensibility/extensions/types.d.ts +12 -0
- package/dist/types/extensibility/gjc-plugins/registry.d.ts +4 -0
- package/dist/types/extensibility/gjc-plugins/runtime-adapters.d.ts +11 -2
- package/dist/types/extensibility/runtime-skill-discovery.d.ts +18 -0
- package/dist/types/extensibility/shared-events.d.ts +3 -1
- package/dist/types/gjc-runtime/session-state-lock.d.ts +61 -0
- package/dist/types/gjc-runtime/state-writer.d.ts +14 -1
- package/dist/types/harness-control-plane/control-endpoint.d.ts +4 -1
- package/dist/types/harness-control-plane/finalize.d.ts +14 -3
- package/dist/types/internal-urls/embedded-protocol.d.ts +9 -0
- package/dist/types/internal-urls/index.d.ts +1 -0
- package/dist/types/internal-urls/parse.d.ts +9 -1
- package/dist/types/main.d.ts +23 -1
- package/dist/types/modes/acp/acp-agent.d.ts +55 -1
- package/dist/types/modes/acp/startup-provenance.d.ts +16 -0
- package/dist/types/modes/components/custom-provider-wizard.d.ts +19 -1
- package/dist/types/modes/components/session-selector.d.ts +5 -2
- package/dist/types/modes/components/user-message-selector.d.ts +5 -2
- package/dist/types/modes/controllers/command-controller.d.ts +1 -1
- package/dist/types/modes/controllers/selector-controller.d.ts +1 -1
- package/dist/types/modes/interactive-mode.d.ts +1 -1
- package/dist/types/modes/shared/agent-wire/workflow-gate-broker.d.ts +11 -1
- package/dist/types/modes/shared/agent-wire/workflow-gate-types.d.ts +14 -1
- package/dist/types/modes/types.d.ts +2 -2
- package/dist/types/runtime/process-lifecycle.d.ts +2 -0
- package/dist/types/runtime-mcp/loader.d.ts +2 -0
- package/dist/types/runtime-mcp/manager.d.ts +69 -6
- package/dist/types/runtime-mcp/pool.d.ts +17 -0
- package/dist/types/runtime-mcp/redaction.d.ts +20 -0
- package/dist/types/runtime-mcp/startup-policy.d.ts +6 -0
- package/dist/types/runtime-mcp/tool-bridge.d.ts +9 -0
- package/dist/types/runtime-mcp/transports/http.d.ts +11 -0
- package/dist/types/sdk/broker/broker.d.ts +17 -4
- package/dist/types/sdk/broker/ensure.d.ts +3 -0
- package/dist/types/sdk/broker/identity.d.ts +5 -1
- package/dist/types/sdk/broker/lifecycle-ledger.d.ts +23 -3
- package/dist/types/sdk/broker/lifecycle.d.ts +15 -8
- package/dist/types/sdk/broker/managed-task-dag.d.ts +1335 -0
- package/dist/types/sdk/broker/managed-task-verification.d.ts +37 -0
- package/dist/types/sdk/broker/runtime.d.ts +1 -1
- package/dist/types/sdk/broker/session-index.d.ts +22 -3
- package/dist/types/sdk/broker/spawn-authority.d.ts +12 -0
- package/dist/types/sdk/broker/startup-failure.d.ts +4 -0
- package/dist/types/sdk/broker/stderr-drainer.d.ts +13 -0
- package/dist/types/sdk/bus/index.d.ts +19 -0
- package/dist/types/sdk/bus/telegram-daemon-contract.d.ts +7 -1
- package/dist/types/sdk/bus/telegram-daemon.d.ts +5 -0
- package/dist/types/sdk/cli/master-cli.d.ts +6 -10
- package/dist/types/sdk/cli/rows.d.ts +19 -1
- package/dist/types/sdk/cli/session-cli.d.ts +22 -5
- package/dist/types/sdk/client/client.d.ts +2 -0
- package/dist/types/sdk/guides/cli.d.ts +3 -4
- package/dist/types/sdk/guides/verify.d.ts +2 -0
- package/dist/types/sdk/host/host.d.ts +9 -0
- package/dist/types/sdk/host/session-runtime.d.ts +31 -0
- package/dist/types/sdk/host/types.d.ts +10 -0
- package/dist/types/sdk/index.d.ts +1 -0
- package/dist/types/sdk/prompt-deadline-flush.d.ts +85 -0
- package/dist/types/sdk/prompt-deadline-manager.d.ts +37 -0
- package/dist/types/sdk/prompt-tool-boundary.d.ts +73 -0
- package/dist/types/sdk/router/session-router.d.ts +2 -0
- package/dist/types/sdk/session.d.ts +3 -3
- package/dist/types/sdk/startup-capability.d.ts +2 -0
- package/dist/types/sdk/transport/relay.d.ts +1 -0
- package/dist/types/sdk/transport/serve-cli.d.ts +48 -4
- package/dist/types/sdk/transport/stdio.d.ts +5 -1
- package/dist/types/session/agent-session.d.ts +175 -23
- package/dist/types/session/fallback-chain-controller.d.ts +6 -6
- package/dist/types/session/fold-coordinator.d.ts +17 -3
- package/dist/types/session/internal/managed-session-storage.d.ts +7 -1
- package/dist/types/session/session-manager.d.ts +64 -1
- package/dist/types/session/session-work-lease.d.ts +14 -0
- package/dist/types/setup/model-onboarding-guidance.d.ts +1 -1
- package/dist/types/setup/provider-onboarding.d.ts +81 -1
- package/dist/types/task/types.d.ts +2 -0
- package/dist/types/tools/bash-interactive.d.ts +1 -0
- package/dist/types/tools/bash.d.ts +37 -11
- package/dist/types/tools/fetch.d.ts +10 -1
- package/dist/types/tools/job.d.ts +1 -1
- package/dist/types/tools/steer-fold.d.ts +31 -0
- package/dist/types/tools/subagent.d.ts +8 -1
- package/dist/types/utils/git.d.ts +28 -2
- package/dist/types/utils/mupdf-embedded.d.ts +2 -0
- package/dist/types/utils/mupdf.d.ts +4 -0
- package/dist/types/utils/session-title-generation.d.ts +6 -0
- package/dist/types/utils/title-generator.d.ts +11 -2
- package/dist/types/web/scrapers/types.d.ts +3 -0
- package/package.json +18 -10
- package/scripts/build-binary.ts +9 -3
- package/scripts/compile-args.ts +0 -9
- package/scripts/embed-mupdf.ts +52 -0
- package/scripts/generate-sdk-operation-inventory.ts +31 -1
- package/scripts/vendor-markit.ts +196 -0
- package/src/async/job-manager.ts +22 -1
- package/src/cli/daemon-cli.ts +53 -18
- package/src/cli/github-release.ts +87 -5
- package/src/cli/mcp-cli.ts +38 -4
- package/src/cli/public-command-entry.ts +450 -0
- package/src/cli/public-command-errors.ts +581 -0
- package/src/cli/public-command-evidence.ts +717 -0
- package/src/cli/public-command-help.ts +360 -0
- package/src/cli/public-command-registry.ts +541 -0
- package/src/cli/session-picker.ts +2 -1
- package/src/cli/setup-cli.ts +10 -1
- package/src/cli/skills-cli.ts +46 -2
- package/src/cli/stats-cli.ts +11 -1
- package/src/cli.ts +61 -1
- package/src/commands/daemon.ts +65 -69
- package/src/commands/sdk.ts +175 -117
- package/src/commands/setup.ts +2 -0
- package/src/commands/skills.ts +11 -0
- package/src/config/autorouting-tier-map.ts +543 -3
- package/src/config/file-lock.ts +206 -65
- package/src/config/model-discovery-manager.ts +4 -4
- package/src/config/model-profile-activation.ts +417 -66
- package/src/config/model-registry.ts +254 -36
- package/src/config/model-resolver.ts +6 -1
- package/src/config/settings-schema.ts +26 -2
- package/src/coordinator-mcp/policy.ts +10 -5
- package/src/coordinator-mcp/question-gate-codec.ts +17 -0
- package/src/coordinator-mcp/question-state.ts +179 -9
- package/src/coordinator-mcp/server.ts +521 -87
- package/src/coordinator-mcp/session-reaper.ts +6 -4
- package/src/crash/record-loader.ts +6 -2
- package/src/daemon/operator-contract.ts +26 -12
- package/src/defaults/gjc/skills/ultragoal/SKILL.md +1 -1
- package/src/eval/js/context-manager.ts +8 -8
- package/src/eval/js/worker-core.ts +9 -4
- package/src/eval/js/worker-protocol.ts +2 -0
- package/src/exec/bash-executor.ts +3 -0
- package/src/extensibility/extensions/runner.ts +52 -4
- package/src/extensibility/extensions/types.ts +10 -0
- package/src/extensibility/gjc-plugins/registry.ts +196 -37
- package/src/extensibility/gjc-plugins/runtime-adapters.ts +46 -8
- package/src/extensibility/runtime-skill-discovery.ts +61 -3
- package/src/extensibility/shared-events.ts +3 -1
- package/src/gjc-runtime/session-state-lock.ts +165 -24
- package/src/gjc-runtime/state-writer.ts +317 -6
- package/src/gjc-runtime/ultragoal-succession.ts +31 -3
- package/src/harness-control-plane/control-endpoint.ts +24 -6
- package/src/harness-control-plane/finalize.ts +312 -25
- package/src/harness-control-plane/owner.ts +323 -89
- package/src/internal-urls/docs-index.generated.ts +23 -20
- package/src/internal-urls/embedded-protocol.ts +83 -0
- package/src/internal-urls/index.ts +1 -0
- package/src/internal-urls/parse.ts +21 -1
- package/src/internal-urls/router.ts +4 -2
- package/src/main.ts +192 -74
- package/src/modes/acp/acp-agent.ts +973 -56
- package/src/modes/acp/startup-provenance.ts +59 -0
- package/src/modes/components/custom-provider-wizard.ts +308 -7
- package/src/modes/components/gjc-bundle-settings.ts +21 -1
- package/src/modes/components/oauth-selector.ts +9 -0
- package/src/modes/components/session-selector.ts +47 -4
- package/src/modes/components/tool-status-header.ts +72 -6
- package/src/modes/components/user-message-selector.ts +57 -15
- package/src/modes/controllers/command-controller.ts +58 -32
- package/src/modes/controllers/extension-ui-controller.ts +33 -7
- package/src/modes/controllers/input-controller.ts +4 -0
- package/src/modes/controllers/runtime-mcp-command-controller.ts +1 -1
- package/src/modes/controllers/selector-controller.ts +161 -15
- package/src/modes/interactive-mode.ts +21 -7
- package/src/modes/runtime-init.ts +1 -0
- package/src/modes/shared/agent-wire/workflow-gate-broker.ts +124 -7
- package/src/modes/shared/agent-wire/workflow-gate-types.ts +16 -2
- package/src/modes/types.ts +2 -2
- package/src/prompts/agent-fragments/browser-aside.md +1 -0
- package/src/prompts/system/title-system.md +1 -1
- package/src/prompts/tools/bash.md +1 -1
- package/src/prompts/tools/job.md +1 -1
- package/src/prompts/tools/read.md +2 -0
- package/src/runtime/network-prewarm-service.test.ts +36 -1
- package/src/runtime/network-prewarm-service.ts +28 -3
- package/src/runtime/process-lifecycle.ts +22 -11
- package/src/runtime-mcp/config.ts +16 -1
- package/src/runtime-mcp/loader.ts +6 -3
- package/src/runtime-mcp/manager.ts +845 -109
- package/src/runtime-mcp/pool.test.ts +31 -0
- package/src/runtime-mcp/pool.ts +304 -28
- package/src/runtime-mcp/redaction.ts +42 -0
- package/src/runtime-mcp/startup-policy.ts +7 -0
- package/src/runtime-mcp/tool-bridge.ts +71 -0
- package/src/runtime-mcp/transports/http.ts +108 -25
- package/src/sdk/acp/adapter.ts +27 -8
- package/src/sdk/broker/broker.ts +1431 -198
- package/src/sdk/broker/ensure.ts +21 -3
- package/src/sdk/broker/identity.ts +22 -1
- package/src/sdk/broker/lifecycle-ledger.ts +95 -31
- package/src/sdk/broker/lifecycle.ts +690 -141
- package/src/sdk/broker/managed-task-dag.ts +1927 -0
- package/src/sdk/broker/managed-task-verification.ts +642 -0
- package/src/sdk/broker/runtime.ts +1 -1
- package/src/sdk/broker/session-index.ts +97 -4
- package/src/sdk/broker/spawn-authority.ts +175 -14
- package/src/sdk/broker/startup-failure.ts +80 -1
- package/src/sdk/broker/stderr-drainer.ts +59 -0
- package/src/sdk/broker/transport.ts +1 -0
- package/src/sdk/bus/chat-daemon-control.ts +2 -2
- package/src/sdk/bus/chat-daemon-runtime.ts +1 -0
- package/src/sdk/bus/index.ts +707 -81
- package/src/sdk/bus/telegram-daemon-cli.ts +31 -2
- package/src/sdk/bus/telegram-daemon-contract.ts +7 -1
- package/src/sdk/bus/telegram-daemon.ts +10 -3
- package/src/sdk/cli/master-cli.ts +47 -36
- package/src/sdk/cli/rows.ts +30 -7
- package/src/sdk/cli/session-cli.ts +705 -184
- package/src/sdk/client/client.ts +18 -0
- package/src/sdk/guides/catalog.ts +2 -2
- package/src/sdk/guides/cli.ts +20 -62
- package/src/sdk/guides/verify.ts +14 -0
- package/src/sdk/host/host.ts +49 -5
- package/src/sdk/host/session-runtime.test.ts +184 -2
- package/src/sdk/host/session-runtime.ts +322 -101
- package/src/sdk/host/types.ts +12 -1
- package/src/sdk/host/websocket-transport.lifecycle.test.ts +84 -0
- package/src/sdk/host/websocket-transport.ts +40 -6
- package/src/sdk/index.ts +11 -0
- package/src/sdk/lifecycle/service.ts +2 -0
- package/src/sdk/prompt-deadline-flush.ts +356 -0
- package/src/sdk/prompt-deadline-manager.ts +133 -1
- package/src/sdk/prompt-tool-boundary.ts +121 -0
- package/src/sdk/protocol/operation-inventory.generated.json +106 -0
- package/src/sdk/router/session-router.ts +9 -1
- package/src/sdk/session.ts +763 -116
- package/src/sdk/startup-capability.ts +1 -1
- package/src/sdk/transport/relay.ts +71 -6
- package/src/sdk/transport/serve-cli.ts +397 -42
- package/src/sdk/transport/socket.ts +12 -11
- package/src/sdk/transport/stdio.ts +7 -3
- package/src/session/agent-session.ts +2472 -407
- package/src/session/fallback-chain-controller.ts +6 -7
- package/src/session/fold-coordinator.ts +24 -5
- package/src/session/internal/managed-session-scope.ts +68 -18
- package/src/session/internal/managed-session-storage.ts +334 -12
- package/src/session/session-manager.ts +382 -44
- package/src/session/session-work-lease.ts +34 -0
- package/src/setup/model-onboarding-guidance.ts +1 -1
- package/src/setup/provider-onboarding.ts +397 -24
- package/src/setup/provider-presets.json +2 -1
- package/src/slash-commands/builtin-registry.ts +259 -15
- package/src/task/executor.ts +46 -17
- package/src/task/index.ts +14 -2
- package/src/task/types.ts +2 -0
- package/src/tools/bash-interactive.ts +2 -0
- package/src/tools/bash.ts +118 -78
- package/src/tools/fetch.ts +140 -25
- package/src/tools/job.ts +151 -25
- package/src/tools/output-meta.ts +15 -2
- package/src/tools/path-utils.ts +27 -3
- package/src/tools/read.ts +3 -3
- package/src/tools/steer-fold.ts +107 -0
- package/src/tools/subagent-render.ts +24 -1
- package/src/tools/subagent.ts +27 -1
- package/src/tools/tool-catalog.generated.ts +29 -2
- package/src/tools/tool-errors.ts +4 -0
- package/src/utils/git.ts +53 -3
- package/src/utils/markit.ts +55 -41
- package/src/utils/mupdf-assets.d.ts +4 -0
- package/src/utils/mupdf-embedded.ts +4 -0
- package/src/utils/mupdf.ts +136 -0
- package/src/utils/session-title-generation.ts +29 -0
- package/src/utils/title-generator.ts +46 -2
- package/src/web/scrapers/types.ts +24 -3
- package/vendor/markit-ai/LICENSE +21 -0
- package/vendor/markit-ai/MANIFEST.json +412 -0
- package/vendor/markit-ai/dist/commands/config.d.ts +4 -0
- package/vendor/markit-ai/dist/commands/config.js +139 -0
- package/vendor/markit-ai/dist/commands/convert.d.ts +6 -0
- package/vendor/markit-ai/dist/commands/convert.js +113 -0
- package/vendor/markit-ai/dist/commands/formats.d.ts +2 -0
- package/vendor/markit-ai/dist/commands/formats.js +92 -0
- package/vendor/markit-ai/dist/commands/init.d.ts +2 -0
- package/vendor/markit-ai/dist/commands/init.js +29 -0
- package/vendor/markit-ai/dist/commands/onboard.d.ts +2 -0
- package/vendor/markit-ai/dist/commands/onboard.js +61 -0
- package/vendor/markit-ai/dist/commands/plugin.d.ts +4 -0
- package/vendor/markit-ai/dist/commands/plugin.js +58 -0
- package/vendor/markit-ai/dist/config.d.ts +26 -0
- package/vendor/markit-ai/dist/config.js +42 -0
- package/vendor/markit-ai/dist/converters/audio.d.ts +7 -0
- package/vendor/markit-ai/dist/converters/audio.js +104 -0
- package/vendor/markit-ai/dist/converters/csv.d.ts +7 -0
- package/vendor/markit-ai/dist/converters/csv.js +83 -0
- package/vendor/markit-ai/dist/converters/docx.d.ts +6 -0
- package/vendor/markit-ai/dist/converters/docx.js +57 -0
- package/vendor/markit-ai/dist/converters/epub.d.ts +8 -0
- package/vendor/markit-ai/dist/converters/epub.js +128 -0
- package/vendor/markit-ai/dist/converters/github.d.ts +18 -0
- package/vendor/markit-ai/dist/converters/github.js +148 -0
- package/vendor/markit-ai/dist/converters/html.d.ts +6 -0
- package/vendor/markit-ai/dist/converters/html.js +30 -0
- package/vendor/markit-ai/dist/converters/image.d.ts +6 -0
- package/vendor/markit-ai/dist/converters/image.js +124 -0
- package/vendor/markit-ai/dist/converters/ipynb.d.ts +6 -0
- package/vendor/markit-ai/dist/converters/ipynb.js +75 -0
- package/vendor/markit-ai/dist/converters/iwork.d.ts +20 -0
- package/vendor/markit-ai/dist/converters/iwork.js +391 -0
- package/vendor/markit-ai/dist/converters/json.d.ts +6 -0
- package/vendor/markit-ai/dist/converters/json.js +21 -0
- package/vendor/markit-ai/dist/converters/pdf/columns.d.ts +35 -0
- package/vendor/markit-ai/dist/converters/pdf/columns.js +93 -0
- package/vendor/markit-ai/dist/converters/pdf/extract.d.ts +19 -0
- package/vendor/markit-ai/dist/converters/pdf/extract.js +514 -0
- package/vendor/markit-ai/dist/converters/pdf/grid.d.ts +25 -0
- package/vendor/markit-ai/dist/converters/pdf/grid.js +795 -0
- package/vendor/markit-ai/dist/converters/pdf/headers.d.ts +24 -0
- package/vendor/markit-ai/dist/converters/pdf/headers.js +108 -0
- package/vendor/markit-ai/dist/converters/pdf/index.d.ts +19 -0
- package/vendor/markit-ai/dist/converters/pdf/index.js +136 -0
- package/vendor/markit-ai/dist/converters/pdf/mupdf-loader.js +9 -0
- package/vendor/markit-ai/dist/converters/pdf/render.d.ts +24 -0
- package/vendor/markit-ai/dist/converters/pdf/render.js +513 -0
- package/vendor/markit-ai/dist/converters/pdf/types.d.ts +75 -0
- package/vendor/markit-ai/dist/converters/pdf/types.js +1 -0
- package/vendor/markit-ai/dist/converters/plain-text.d.ts +6 -0
- package/vendor/markit-ai/dist/converters/plain-text.js +87 -0
- package/vendor/markit-ai/dist/converters/pptx.d.ts +8 -0
- package/vendor/markit-ai/dist/converters/pptx.js +289 -0
- package/vendor/markit-ai/dist/converters/rss.d.ts +11 -0
- package/vendor/markit-ai/dist/converters/rss.js +138 -0
- package/vendor/markit-ai/dist/converters/wikipedia.d.ts +6 -0
- package/vendor/markit-ai/dist/converters/wikipedia.js +33 -0
- package/vendor/markit-ai/dist/converters/xlsx.d.ts +8 -0
- package/vendor/markit-ai/dist/converters/xlsx.js +147 -0
- package/vendor/markit-ai/dist/converters/xml.d.ts +6 -0
- package/vendor/markit-ai/dist/converters/xml.js +18 -0
- package/vendor/markit-ai/dist/converters/yaml.d.ts +6 -0
- package/vendor/markit-ai/dist/converters/yaml.js +17 -0
- package/vendor/markit-ai/dist/converters/zip.d.ts +8 -0
- package/vendor/markit-ai/dist/converters/zip.js +57 -0
- package/vendor/markit-ai/dist/index.d.ts +30 -0
- package/vendor/markit-ai/dist/index.js +26 -0
- package/vendor/markit-ai/dist/main.d.ts +2 -0
- package/vendor/markit-ai/dist/main.js +191 -0
- package/vendor/markit-ai/dist/markit.d.ts +45 -0
- package/vendor/markit-ai/dist/markit.js +267 -0
- package/vendor/markit-ai/dist/plugins/api.d.ts +7 -0
- package/vendor/markit-ai/dist/plugins/api.js +42 -0
- package/vendor/markit-ai/dist/plugins/index.d.ts +4 -0
- package/vendor/markit-ai/dist/plugins/index.js +3 -0
- package/vendor/markit-ai/dist/plugins/installer.d.ts +25 -0
- package/vendor/markit-ai/dist/plugins/installer.js +176 -0
- package/vendor/markit-ai/dist/plugins/loader.d.ts +6 -0
- package/vendor/markit-ai/dist/plugins/loader.js +61 -0
- package/vendor/markit-ai/dist/plugins/types.d.ts +25 -0
- package/vendor/markit-ai/dist/plugins/types.js +1 -0
- package/vendor/markit-ai/dist/providers/anthropic.d.ts +2 -0
- package/vendor/markit-ai/dist/providers/anthropic.js +47 -0
- package/vendor/markit-ai/dist/providers/index.d.ts +21 -0
- package/vendor/markit-ai/dist/providers/index.js +56 -0
- package/vendor/markit-ai/dist/providers/openai.d.ts +2 -0
- package/vendor/markit-ai/dist/providers/openai.js +76 -0
- package/vendor/markit-ai/dist/providers/types.d.ts +26 -0
- package/vendor/markit-ai/dist/providers/types.js +1 -0
- package/vendor/markit-ai/dist/types.d.ts +36 -0
- package/vendor/markit-ai/dist/types.js +1 -0
- package/vendor/markit-ai/dist/utils/exit-codes.d.ts +4 -0
- package/vendor/markit-ai/dist/utils/exit-codes.js +4 -0
- package/vendor/markit-ai/dist/utils/output.d.ts +22 -0
- package/vendor/markit-ai/dist/utils/output.js +31 -0
- package/vendor/markit-ai/dist/utils/turndown.d.ts +8 -0
- package/vendor/markit-ai/dist/utils/turndown.js +70 -0
- package/vendor/markit-ai/package.json +74 -0
- package/vendor/markit-ai.patch +49 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
const EXTENSIONS = [".yaml", ".yml"];
|
|
2
|
+
const MIMETYPES = ["text/yaml", "application/x-yaml"];
|
|
3
|
+
export class YamlConverter {
|
|
4
|
+
name = "yaml";
|
|
5
|
+
accepts(streamInfo) {
|
|
6
|
+
if (streamInfo.extension && EXTENSIONS.includes(streamInfo.extension))
|
|
7
|
+
return true;
|
|
8
|
+
if (streamInfo.mimetype &&
|
|
9
|
+
MIMETYPES.some((m) => streamInfo.mimetype?.startsWith(m)))
|
|
10
|
+
return true;
|
|
11
|
+
return false;
|
|
12
|
+
}
|
|
13
|
+
async convert(input, streamInfo) {
|
|
14
|
+
const text = new TextDecoder(streamInfo.charset || "utf-8").decode(input);
|
|
15
|
+
return { markdown: `\`\`\`yaml\n${text}\n\`\`\`` };
|
|
16
|
+
}
|
|
17
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ConversionResult, Converter, StreamInfo } from "../types.js";
|
|
2
|
+
export declare class ZipConverter implements Converter {
|
|
3
|
+
name: string;
|
|
4
|
+
private parentConverters;
|
|
5
|
+
constructor(parentConverters: Converter[]);
|
|
6
|
+
accepts(streamInfo: StreamInfo): boolean;
|
|
7
|
+
convert(input: Buffer, streamInfo: StreamInfo): Promise<ConversionResult>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { basename, extname } from "node:path";
|
|
2
|
+
import JSZip from "jszip";
|
|
3
|
+
const EXTENSIONS = [".zip"];
|
|
4
|
+
const MIMETYPES = ["application/zip", "application/x-zip-compressed"];
|
|
5
|
+
export class ZipConverter {
|
|
6
|
+
name = "zip";
|
|
7
|
+
parentConverters;
|
|
8
|
+
constructor(parentConverters) {
|
|
9
|
+
this.parentConverters = parentConverters;
|
|
10
|
+
}
|
|
11
|
+
accepts(streamInfo) {
|
|
12
|
+
if (streamInfo.extension && EXTENSIONS.includes(streamInfo.extension))
|
|
13
|
+
return true;
|
|
14
|
+
if (streamInfo.mimetype &&
|
|
15
|
+
MIMETYPES.some((m) => streamInfo.mimetype?.startsWith(m)))
|
|
16
|
+
return true;
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
async convert(input, streamInfo) {
|
|
20
|
+
const zip = await JSZip.loadAsync(input);
|
|
21
|
+
const label = streamInfo.localPath || streamInfo.filename || "archive.zip";
|
|
22
|
+
const sections = [`Content from \`${basename(label)}\`:`];
|
|
23
|
+
for (const [path, file] of Object.entries(zip.files)) {
|
|
24
|
+
if (file.dir)
|
|
25
|
+
continue;
|
|
26
|
+
const ext = extname(path).toLowerCase();
|
|
27
|
+
const fileInfo = {
|
|
28
|
+
extension: ext,
|
|
29
|
+
filename: basename(path),
|
|
30
|
+
};
|
|
31
|
+
const buffer = Buffer.from(await file.async("arraybuffer"));
|
|
32
|
+
// Try each converter
|
|
33
|
+
let converted = false;
|
|
34
|
+
for (const converter of this.parentConverters) {
|
|
35
|
+
if (converter.name === "zip")
|
|
36
|
+
continue; // avoid recursion loops
|
|
37
|
+
if (!converter.accepts(fileInfo))
|
|
38
|
+
continue;
|
|
39
|
+
try {
|
|
40
|
+
const result = await converter.convert(buffer, fileInfo);
|
|
41
|
+
if (result.markdown.trim()) {
|
|
42
|
+
sections.push(`## File: ${path}\n\n${result.markdown.trim()}`);
|
|
43
|
+
converted = true;
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
// Try next converter
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
if (!converted) {
|
|
52
|
+
sections.push(`## File: ${path}\n\n*[binary file]*`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return { markdown: sections.join("\n\n").trim() };
|
|
56
|
+
}
|
|
57
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export type { MarkitConfig } from "./config.js";
|
|
2
|
+
export { AudioConverter } from "./converters/audio.js";
|
|
3
|
+
export { CsvConverter } from "./converters/csv.js";
|
|
4
|
+
export { DocxConverter } from "./converters/docx.js";
|
|
5
|
+
export { EpubConverter } from "./converters/epub.js";
|
|
6
|
+
export { GitHubConverter } from "./converters/github.js";
|
|
7
|
+
export { HtmlConverter } from "./converters/html.js";
|
|
8
|
+
export { ImageConverter } from "./converters/image.js";
|
|
9
|
+
export { IpynbConverter } from "./converters/ipynb.js";
|
|
10
|
+
export { IWorkConverter } from "./converters/iwork.js";
|
|
11
|
+
export { JsonConverter } from "./converters/json.js";
|
|
12
|
+
export { PdfConverter } from "./converters/pdf/index.js";
|
|
13
|
+
export { PlainTextConverter } from "./converters/plain-text.js";
|
|
14
|
+
export { PptxConverter } from "./converters/pptx.js";
|
|
15
|
+
export { RssConverter } from "./converters/rss.js";
|
|
16
|
+
export { WikipediaConverter } from "./converters/wikipedia.js";
|
|
17
|
+
export { XlsxConverter } from "./converters/xlsx.js";
|
|
18
|
+
export { XmlConverter } from "./converters/xml.js";
|
|
19
|
+
export { YamlConverter } from "./converters/yaml.js";
|
|
20
|
+
export { ZipConverter } from "./converters/zip.js";
|
|
21
|
+
export { Markit } from "./markit.js";
|
|
22
|
+
export { createPluginAPI, resolvePluginExport } from "./plugins/api.js";
|
|
23
|
+
export { installPlugin, listInstalled, removePlugin, } from "./plugins/installer.js";
|
|
24
|
+
export { loadAllPlugins, loadPluginFromPath } from "./plugins/loader.js";
|
|
25
|
+
export type { MarkitPluginAPI, PluginDef, PluginFunction, } from "./plugins/types.js";
|
|
26
|
+
export { anthropic } from "./providers/anthropic.js";
|
|
27
|
+
export { createLlmFunctions, getProvider, listProviders, registerProvider, } from "./providers/index.js";
|
|
28
|
+
export { openai } from "./providers/openai.js";
|
|
29
|
+
export type { Provider, ProviderConfig, ResolvedConfig, } from "./providers/types.js";
|
|
30
|
+
export type { ConversionResult, Converter, MarkitOptions, StreamInfo, } from "./types.js";
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export { AudioConverter } from "./converters/audio.js";
|
|
2
|
+
export { CsvConverter } from "./converters/csv.js";
|
|
3
|
+
export { DocxConverter } from "./converters/docx.js";
|
|
4
|
+
export { EpubConverter } from "./converters/epub.js";
|
|
5
|
+
export { GitHubConverter } from "./converters/github.js";
|
|
6
|
+
export { HtmlConverter } from "./converters/html.js";
|
|
7
|
+
export { ImageConverter } from "./converters/image.js";
|
|
8
|
+
export { IpynbConverter } from "./converters/ipynb.js";
|
|
9
|
+
export { IWorkConverter } from "./converters/iwork.js";
|
|
10
|
+
export { JsonConverter } from "./converters/json.js";
|
|
11
|
+
export { PdfConverter } from "./converters/pdf/index.js";
|
|
12
|
+
export { PlainTextConverter } from "./converters/plain-text.js";
|
|
13
|
+
export { PptxConverter } from "./converters/pptx.js";
|
|
14
|
+
export { RssConverter } from "./converters/rss.js";
|
|
15
|
+
export { WikipediaConverter } from "./converters/wikipedia.js";
|
|
16
|
+
export { XlsxConverter } from "./converters/xlsx.js";
|
|
17
|
+
export { XmlConverter } from "./converters/xml.js";
|
|
18
|
+
export { YamlConverter } from "./converters/yaml.js";
|
|
19
|
+
export { ZipConverter } from "./converters/zip.js";
|
|
20
|
+
export { Markit } from "./markit.js";
|
|
21
|
+
export { createPluginAPI, resolvePluginExport } from "./plugins/api.js";
|
|
22
|
+
export { installPlugin, listInstalled, removePlugin, } from "./plugins/installer.js";
|
|
23
|
+
export { loadAllPlugins, loadPluginFromPath } from "./plugins/loader.js";
|
|
24
|
+
export { anthropic } from "./providers/anthropic.js";
|
|
25
|
+
export { createLlmFunctions, getProvider, listProviders, registerProvider, } from "./providers/index.js";
|
|
26
|
+
export { openai } from "./providers/openai.js";
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { createRequire } from "node:module";
|
|
3
|
+
import { Command } from "commander";
|
|
4
|
+
import { configGet, configSet, configShow } from "./commands/config.js";
|
|
5
|
+
import { convert } from "./commands/convert.js";
|
|
6
|
+
import { formats } from "./commands/formats.js";
|
|
7
|
+
import { init } from "./commands/init.js";
|
|
8
|
+
import { onboard } from "./commands/onboard.js";
|
|
9
|
+
import { pluginInstall, pluginList, pluginRemove } from "./commands/plugin.js";
|
|
10
|
+
const require = createRequire(import.meta.url);
|
|
11
|
+
const { version } = require("../package.json");
|
|
12
|
+
const program = new Command();
|
|
13
|
+
program
|
|
14
|
+
.name("markit")
|
|
15
|
+
.description("Convert anything to markdown.")
|
|
16
|
+
.version(`markit ${version}`, "-V, --version")
|
|
17
|
+
.option("--json", "Output as JSON")
|
|
18
|
+
.option("-q, --quiet", "Raw markdown only, no decoration")
|
|
19
|
+
.option("-p, --prompt <text>", "Extra instructions for image description")
|
|
20
|
+
.option("-o, --output <file>", "Write to file instead of stdout")
|
|
21
|
+
.option("-i, --image-dir <dir>", "Extract images to this directory")
|
|
22
|
+
.addHelpText("after", `
|
|
23
|
+
Examples:
|
|
24
|
+
$ markit report.pdf Convert a PDF to markdown
|
|
25
|
+
$ markit document.docx -o doc.md Convert DOCX, write to file
|
|
26
|
+
$ markit https://example.com Convert a web page
|
|
27
|
+
$ markit photo.jpg Extract EXIF + AI description
|
|
28
|
+
$ markit recording.mp3 Metadata + transcription
|
|
29
|
+
$ cat file.pdf | markit - Read from stdin
|
|
30
|
+
$ markit init Create .markit/ config
|
|
31
|
+
$ markit config show Show LLM settings
|
|
32
|
+
|
|
33
|
+
Docs: https://github.com/Michaelliv/markit`);
|
|
34
|
+
program
|
|
35
|
+
.command("convert")
|
|
36
|
+
.alias("c")
|
|
37
|
+
.description("Convert a file or URL to markdown")
|
|
38
|
+
.argument("<source>", "File path, URL, or - for stdin")
|
|
39
|
+
.option("-o, --output <file>", "Write to file instead of stdout")
|
|
40
|
+
.action(async (source, opts, cmd) => {
|
|
41
|
+
const globals = cmd.optsWithGlobals();
|
|
42
|
+
await convert(source, {
|
|
43
|
+
json: globals.json,
|
|
44
|
+
quiet: globals.quiet,
|
|
45
|
+
output: opts.output,
|
|
46
|
+
prompt: globals.prompt,
|
|
47
|
+
imageDir: globals.imageDir,
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
program
|
|
51
|
+
.command("init")
|
|
52
|
+
.description("Create .markit/ config directory")
|
|
53
|
+
.action(async (_opts, cmd) => {
|
|
54
|
+
const globals = cmd.optsWithGlobals();
|
|
55
|
+
await init([], { json: globals.json, quiet: globals.quiet });
|
|
56
|
+
});
|
|
57
|
+
const configCmd = program
|
|
58
|
+
.command("config")
|
|
59
|
+
.description("Manage markit configuration");
|
|
60
|
+
configCmd
|
|
61
|
+
.command("show")
|
|
62
|
+
.description("Show current configuration")
|
|
63
|
+
.action(async (_opts, cmd) => {
|
|
64
|
+
const globals = cmd.optsWithGlobals();
|
|
65
|
+
await configShow([], { json: globals.json, quiet: globals.quiet });
|
|
66
|
+
});
|
|
67
|
+
configCmd
|
|
68
|
+
.command("get <key>")
|
|
69
|
+
.description("Get a config value")
|
|
70
|
+
.action(async (key, _opts, cmd) => {
|
|
71
|
+
const globals = cmd.optsWithGlobals();
|
|
72
|
+
await configGet(key, { json: globals.json, quiet: globals.quiet });
|
|
73
|
+
});
|
|
74
|
+
configCmd
|
|
75
|
+
.command("set <key> [value]")
|
|
76
|
+
.description("Set a config value (secrets read from stdin if no value given)")
|
|
77
|
+
.action(async (key, value, _opts, cmd) => {
|
|
78
|
+
const globals = cmd.optsWithGlobals();
|
|
79
|
+
await configSet(key, value, { json: globals.json, quiet: globals.quiet });
|
|
80
|
+
});
|
|
81
|
+
const pluginCmd = program.command("plugin").description("Manage plugins");
|
|
82
|
+
pluginCmd
|
|
83
|
+
.command("install <source>")
|
|
84
|
+
.description("Install a plugin (npm:pkg, git:url, or local path)")
|
|
85
|
+
.action(async (source, _opts, cmd) => {
|
|
86
|
+
const globals = cmd.optsWithGlobals();
|
|
87
|
+
await pluginInstall(source, { json: globals.json, quiet: globals.quiet });
|
|
88
|
+
});
|
|
89
|
+
pluginCmd
|
|
90
|
+
.command("remove <name>")
|
|
91
|
+
.description("Remove an installed plugin")
|
|
92
|
+
.action(async (name, _opts, cmd) => {
|
|
93
|
+
const globals = cmd.optsWithGlobals();
|
|
94
|
+
await pluginRemove(name, { json: globals.json, quiet: globals.quiet });
|
|
95
|
+
});
|
|
96
|
+
pluginCmd
|
|
97
|
+
.command("list")
|
|
98
|
+
.description("List installed plugins")
|
|
99
|
+
.action(async (_opts, cmd) => {
|
|
100
|
+
const globals = cmd.optsWithGlobals();
|
|
101
|
+
await pluginList({ json: globals.json, quiet: globals.quiet });
|
|
102
|
+
});
|
|
103
|
+
program
|
|
104
|
+
.command("formats")
|
|
105
|
+
.description("List supported formats")
|
|
106
|
+
.action(async (_opts, cmd) => {
|
|
107
|
+
const globals = cmd.optsWithGlobals();
|
|
108
|
+
await formats([], { json: globals.json, quiet: globals.quiet });
|
|
109
|
+
});
|
|
110
|
+
program
|
|
111
|
+
.command("onboard")
|
|
112
|
+
.description("Add markit instructions to CLAUDE.md or AGENTS.md")
|
|
113
|
+
.action(async (_opts, cmd) => {
|
|
114
|
+
const globals = cmd.optsWithGlobals();
|
|
115
|
+
await onboard([], { json: globals.json, quiet: globals.quiet });
|
|
116
|
+
});
|
|
117
|
+
// Default behavior: if first arg isn't a known subcommand, treat it as a source to convert
|
|
118
|
+
program.on("command:*", async (args) => {
|
|
119
|
+
const source = args[0];
|
|
120
|
+
if (!source) {
|
|
121
|
+
program.help();
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
// Check for typos against known subcommands
|
|
125
|
+
const commands = [
|
|
126
|
+
"convert",
|
|
127
|
+
"formats",
|
|
128
|
+
"onboard",
|
|
129
|
+
"help",
|
|
130
|
+
"init",
|
|
131
|
+
"config",
|
|
132
|
+
"plugin",
|
|
133
|
+
];
|
|
134
|
+
const close = commands.filter((c) => levenshtein(source, c) <= 2 && source !== c);
|
|
135
|
+
if (close.length > 0 &&
|
|
136
|
+
!source.includes("/") &&
|
|
137
|
+
!source.includes(".") &&
|
|
138
|
+
!source.startsWith("http")) {
|
|
139
|
+
const { error } = await import("./utils/output.js");
|
|
140
|
+
error(`Unknown command '${source}'. Did you mean '${close[0]}'?`);
|
|
141
|
+
process.exit(1);
|
|
142
|
+
}
|
|
143
|
+
const globals = program.opts();
|
|
144
|
+
await convert(source, {
|
|
145
|
+
json: globals.json,
|
|
146
|
+
quiet: globals.quiet,
|
|
147
|
+
output: globals.output,
|
|
148
|
+
prompt: globals.prompt,
|
|
149
|
+
imageDir: globals.imageDir,
|
|
150
|
+
});
|
|
151
|
+
});
|
|
152
|
+
// No args → show concise help
|
|
153
|
+
if (process.argv.length <= 2) {
|
|
154
|
+
console.log(`markit — convert anything to markdown
|
|
155
|
+
|
|
156
|
+
Usage: markit <file-or-url> [options]
|
|
157
|
+
|
|
158
|
+
Examples:
|
|
159
|
+
$ markit report.pdf
|
|
160
|
+
$ markit document.docx -o doc.md
|
|
161
|
+
$ markit https://example.com
|
|
162
|
+
|
|
163
|
+
Commands:
|
|
164
|
+
markit init Create .markit/ config directory
|
|
165
|
+
markit config Manage settings (LLM, API keys)
|
|
166
|
+
markit formats List supported formats
|
|
167
|
+
markit onboard Add instructions to CLAUDE.md
|
|
168
|
+
|
|
169
|
+
Run markit --help for all options.
|
|
170
|
+
Docs: https://github.com/Michaelliv/markit`);
|
|
171
|
+
process.exit(0);
|
|
172
|
+
}
|
|
173
|
+
program.parseAsync(process.argv).catch((err) => {
|
|
174
|
+
console.error("Fatal error:", err.message);
|
|
175
|
+
process.exit(1);
|
|
176
|
+
});
|
|
177
|
+
function levenshtein(a, b) {
|
|
178
|
+
const m = a.length;
|
|
179
|
+
const n = b.length;
|
|
180
|
+
const dp = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0));
|
|
181
|
+
for (let i = 0; i <= m; i++)
|
|
182
|
+
dp[i][0] = i;
|
|
183
|
+
for (let j = 0; j <= n; j++)
|
|
184
|
+
dp[0][j] = j;
|
|
185
|
+
for (let i = 1; i <= m; i++) {
|
|
186
|
+
for (let j = 1; j <= n; j++) {
|
|
187
|
+
dp[i][j] = Math.min(dp[i - 1][j] + 1, dp[i][j - 1] + 1, dp[i - 1][j - 1] + (a[i - 1] !== b[j - 1] ? 1 : 0));
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
return dp[m][n];
|
|
191
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { PluginDef } from "./plugins/types.js";
|
|
2
|
+
import type { ConversionResult, MarkitOptions, StreamInfo } from "./types.js";
|
|
3
|
+
export declare class Markit {
|
|
4
|
+
private converters;
|
|
5
|
+
private options;
|
|
6
|
+
constructor(options?: MarkitOptions, plugins?: PluginDef[]);
|
|
7
|
+
/**
|
|
8
|
+
* Convert a local file to markdown.
|
|
9
|
+
*/
|
|
10
|
+
convertFile(path: string, extra?: Partial<StreamInfo>): Promise<ConversionResult>;
|
|
11
|
+
/**
|
|
12
|
+
* Convert a URL to markdown.
|
|
13
|
+
*/
|
|
14
|
+
convertUrl(url: string): Promise<ConversionResult>;
|
|
15
|
+
/**
|
|
16
|
+
* For root URLs, check if the site publishes /llms.txt.
|
|
17
|
+
* If it exists, return it as markdown directly.
|
|
18
|
+
*/
|
|
19
|
+
private tryLlmsTxt;
|
|
20
|
+
/**
|
|
21
|
+
* Inspect an HTML response for a discoverable markdown source URL.
|
|
22
|
+
* If found, fetch and convert the raw markdown instead.
|
|
23
|
+
*/
|
|
24
|
+
private tryMarkdownSource;
|
|
25
|
+
/**
|
|
26
|
+
* Fetch a markdown source URL, validating the response is actually markdown.
|
|
27
|
+
*/
|
|
28
|
+
private fetchMarkdownSource;
|
|
29
|
+
/**
|
|
30
|
+
* Convert a buffer with stream info to markdown.
|
|
31
|
+
*/
|
|
32
|
+
convert(input: Buffer, streamInfo: StreamInfo): Promise<ConversionResult>;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Try to discover a raw markdown source URL from an HTML response.
|
|
36
|
+
* Checks for known markers in the HTML itself:
|
|
37
|
+
* 1. <link rel="alternate" type="text/markdown" href="..."> tag
|
|
38
|
+
* 2. VitePress markers → append .md to the URL
|
|
39
|
+
*
|
|
40
|
+
* The llms.txt .md probe is handled separately in tryMarkdownSource
|
|
41
|
+
* as a fallback when no markers are found.
|
|
42
|
+
*
|
|
43
|
+
* @internal Exported for testing.
|
|
44
|
+
*/
|
|
45
|
+
export declare function discoverMarkdownSource(html: string, url: string, ext: string): string | null;
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { basename, extname } from "node:path";
|
|
3
|
+
import { AudioConverter } from "./converters/audio.js";
|
|
4
|
+
import { CsvConverter } from "./converters/csv.js";
|
|
5
|
+
import { DocxConverter } from "./converters/docx.js";
|
|
6
|
+
import { EpubConverter } from "./converters/epub.js";
|
|
7
|
+
import { GitHubConverter } from "./converters/github.js";
|
|
8
|
+
import { HtmlConverter } from "./converters/html.js";
|
|
9
|
+
import { ImageConverter } from "./converters/image.js";
|
|
10
|
+
import { IpynbConverter } from "./converters/ipynb.js";
|
|
11
|
+
import { IWorkConverter } from "./converters/iwork.js";
|
|
12
|
+
import { JsonConverter } from "./converters/json.js";
|
|
13
|
+
import { PdfConverter } from "./converters/pdf/index.js";
|
|
14
|
+
import { PlainTextConverter } from "./converters/plain-text.js";
|
|
15
|
+
import { PptxConverter } from "./converters/pptx.js";
|
|
16
|
+
import { RssConverter } from "./converters/rss.js";
|
|
17
|
+
import { WikipediaConverter } from "./converters/wikipedia.js";
|
|
18
|
+
import { XlsxConverter } from "./converters/xlsx.js";
|
|
19
|
+
import { XmlConverter } from "./converters/xml.js";
|
|
20
|
+
import { YamlConverter } from "./converters/yaml.js";
|
|
21
|
+
import { ZipConverter } from "./converters/zip.js";
|
|
22
|
+
const USER_AGENT = "markit/0.1.0";
|
|
23
|
+
export class Markit {
|
|
24
|
+
converters = [];
|
|
25
|
+
options;
|
|
26
|
+
constructor(options = {}, plugins = []) {
|
|
27
|
+
this.options = options;
|
|
28
|
+
// Plugin converters go first — they override builtins for the same format
|
|
29
|
+
const pluginConverters = plugins.flatMap((p) => p.converters);
|
|
30
|
+
// Built-in converters: specific formats first, generic last.
|
|
31
|
+
const specific = [
|
|
32
|
+
new PdfConverter(),
|
|
33
|
+
new DocxConverter(),
|
|
34
|
+
new PptxConverter(),
|
|
35
|
+
new XlsxConverter(),
|
|
36
|
+
new EpubConverter(),
|
|
37
|
+
new IpynbConverter(),
|
|
38
|
+
new IWorkConverter(),
|
|
39
|
+
new GitHubConverter(),
|
|
40
|
+
new WikipediaConverter(),
|
|
41
|
+
new RssConverter(),
|
|
42
|
+
new CsvConverter(),
|
|
43
|
+
new JsonConverter(),
|
|
44
|
+
new YamlConverter(),
|
|
45
|
+
new ImageConverter(),
|
|
46
|
+
new AudioConverter(),
|
|
47
|
+
];
|
|
48
|
+
const generic = [new XmlConverter(), new HtmlConverter()];
|
|
49
|
+
// ZIP gets all converters (plugin + builtin) for recursive extraction
|
|
50
|
+
const allNonZip = [...pluginConverters, ...specific, ...generic];
|
|
51
|
+
const zipConverter = new ZipConverter(allNonZip);
|
|
52
|
+
// Plugin converters first, then builtins, plain text last
|
|
53
|
+
this.converters = [
|
|
54
|
+
...pluginConverters,
|
|
55
|
+
...specific,
|
|
56
|
+
zipConverter,
|
|
57
|
+
...generic,
|
|
58
|
+
new PlainTextConverter(),
|
|
59
|
+
];
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Convert a local file to markdown.
|
|
63
|
+
*/
|
|
64
|
+
async convertFile(path, extra) {
|
|
65
|
+
const buffer = readFileSync(path);
|
|
66
|
+
const streamInfo = {
|
|
67
|
+
localPath: path,
|
|
68
|
+
extension: extname(path).toLowerCase(),
|
|
69
|
+
filename: basename(path),
|
|
70
|
+
...extra,
|
|
71
|
+
};
|
|
72
|
+
return this.convert(buffer, streamInfo);
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Convert a URL to markdown.
|
|
76
|
+
*/
|
|
77
|
+
async convertUrl(url) {
|
|
78
|
+
// Let converters with a URL-specific hook handle it first
|
|
79
|
+
const streamInfo = { url };
|
|
80
|
+
for (const converter of this.converters) {
|
|
81
|
+
if (!converter.convertUrl || !converter.accepts(streamInfo))
|
|
82
|
+
continue;
|
|
83
|
+
try {
|
|
84
|
+
return await converter.convertUrl(url, this.options);
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
// Fall through to default fetch path
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
// For root URLs, check if the site has /llms.txt and return it if so
|
|
91
|
+
const parsedUrl = new URL(url);
|
|
92
|
+
if (parsedUrl.pathname === "/" || parsedUrl.pathname === "") {
|
|
93
|
+
const result = await this.tryLlmsTxt(parsedUrl.origin);
|
|
94
|
+
if (result)
|
|
95
|
+
return result;
|
|
96
|
+
}
|
|
97
|
+
const response = await fetch(url, {
|
|
98
|
+
headers: {
|
|
99
|
+
Accept: "text/markdown, text/html;q=0.9, text/plain;q=0.8, */*;q=0.1",
|
|
100
|
+
"User-Agent": USER_AGENT,
|
|
101
|
+
},
|
|
102
|
+
});
|
|
103
|
+
if (!response.ok) {
|
|
104
|
+
throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
|
|
105
|
+
}
|
|
106
|
+
const contentType = response.headers.get("content-type") || "";
|
|
107
|
+
const mimetype = contentType.split(";")[0].trim();
|
|
108
|
+
const urlPath = new URL(url).pathname;
|
|
109
|
+
const ext = extname(urlPath).toLowerCase();
|
|
110
|
+
// Content negotiation worked — server returned markdown directly
|
|
111
|
+
if (mimetype === "text/markdown") {
|
|
112
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
113
|
+
return this.convert(buffer, {
|
|
114
|
+
url,
|
|
115
|
+
mimetype: "text/markdown",
|
|
116
|
+
extension: ".md",
|
|
117
|
+
filename: basename(urlPath) || undefined,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
121
|
+
// For HTML responses, try to discover a raw markdown source.
|
|
122
|
+
// Patterns: <link rel="alternate">, VitePress .md files, llms.txt convention.
|
|
123
|
+
if (mimetype === "text/html") {
|
|
124
|
+
const result = await this.tryMarkdownSource(buffer, url, ext);
|
|
125
|
+
if (result)
|
|
126
|
+
return result;
|
|
127
|
+
}
|
|
128
|
+
return this.convert(buffer, {
|
|
129
|
+
url,
|
|
130
|
+
mimetype,
|
|
131
|
+
extension: ext || undefined,
|
|
132
|
+
filename: basename(urlPath) || undefined,
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* For root URLs, check if the site publishes /llms.txt.
|
|
137
|
+
* If it exists, return it as markdown directly.
|
|
138
|
+
*/
|
|
139
|
+
async tryLlmsTxt(origin) {
|
|
140
|
+
const llmsTxtUrl = `${origin}/llms.txt`;
|
|
141
|
+
try {
|
|
142
|
+
const response = await fetch(llmsTxtUrl, {
|
|
143
|
+
method: "HEAD",
|
|
144
|
+
headers: { "User-Agent": USER_AGENT },
|
|
145
|
+
});
|
|
146
|
+
if (!response.ok)
|
|
147
|
+
return null;
|
|
148
|
+
const ct = (response.headers.get("content-type") || "")
|
|
149
|
+
.split(";")[0]
|
|
150
|
+
.trim();
|
|
151
|
+
if (!ct.includes("markdown") &&
|
|
152
|
+
!ct.includes("text/plain") &&
|
|
153
|
+
!ct.includes("text/html"))
|
|
154
|
+
return null;
|
|
155
|
+
// HEAD succeeded — now GET the content
|
|
156
|
+
const getResponse = await fetch(llmsTxtUrl, {
|
|
157
|
+
headers: { "User-Agent": USER_AGENT },
|
|
158
|
+
});
|
|
159
|
+
if (!getResponse.ok)
|
|
160
|
+
return null;
|
|
161
|
+
const buffer = Buffer.from(await getResponse.arrayBuffer());
|
|
162
|
+
return { markdown: buffer.toString("utf-8") };
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Inspect an HTML response for a discoverable markdown source URL.
|
|
170
|
+
* If found, fetch and convert the raw markdown instead.
|
|
171
|
+
*/
|
|
172
|
+
async tryMarkdownSource(htmlBuffer, url, ext) {
|
|
173
|
+
const html = htmlBuffer.toString("utf-8", 0, Math.min(htmlBuffer.length, 50_000));
|
|
174
|
+
const mdSourceUrl = discoverMarkdownSource(html, url, ext);
|
|
175
|
+
if (!mdSourceUrl)
|
|
176
|
+
return null;
|
|
177
|
+
return this.fetchMarkdownSource(mdSourceUrl);
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Fetch a markdown source URL, validating the response is actually markdown.
|
|
181
|
+
*/
|
|
182
|
+
async fetchMarkdownSource(mdUrl) {
|
|
183
|
+
try {
|
|
184
|
+
const response = await fetch(mdUrl, {
|
|
185
|
+
headers: { "User-Agent": USER_AGENT },
|
|
186
|
+
});
|
|
187
|
+
if (!response.ok)
|
|
188
|
+
return null;
|
|
189
|
+
const ct = (response.headers.get("content-type") || "")
|
|
190
|
+
.split(";")[0]
|
|
191
|
+
.trim();
|
|
192
|
+
if (!ct.includes("markdown") && !ct.includes("text/plain"))
|
|
193
|
+
return null;
|
|
194
|
+
const mdBuffer = Buffer.from(await response.arrayBuffer());
|
|
195
|
+
return this.convert(mdBuffer, {
|
|
196
|
+
url: mdUrl,
|
|
197
|
+
mimetype: "text/markdown",
|
|
198
|
+
extension: ".md",
|
|
199
|
+
filename: basename(new URL(mdUrl).pathname),
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
catch {
|
|
203
|
+
return null;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Convert a buffer with stream info to markdown.
|
|
208
|
+
*/
|
|
209
|
+
async convert(input, streamInfo) {
|
|
210
|
+
const errors = [];
|
|
211
|
+
for (const converter of this.converters) {
|
|
212
|
+
if (!converter.accepts(streamInfo))
|
|
213
|
+
continue;
|
|
214
|
+
try {
|
|
215
|
+
return await converter.convert(input, streamInfo, this.options);
|
|
216
|
+
}
|
|
217
|
+
catch (err) {
|
|
218
|
+
errors.push({
|
|
219
|
+
converter: converter.name,
|
|
220
|
+
error: err instanceof Error ? err : new Error(String(err)),
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
if (errors.length > 0) {
|
|
225
|
+
const details = errors
|
|
226
|
+
.map((e) => ` ${e.converter}: ${e.error.message}`)
|
|
227
|
+
.join("\n");
|
|
228
|
+
throw new AggregateError(errors.map((entry) => entry.error), `Conversion failed:\n${details}`, { cause: errors[0].error });
|
|
229
|
+
}
|
|
230
|
+
throw new Error(`Unsupported format: ${streamInfo.extension || streamInfo.mimetype || "unknown"}`);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Try to discover a raw markdown source URL from an HTML response.
|
|
235
|
+
* Checks for known markers in the HTML itself:
|
|
236
|
+
* 1. <link rel="alternate" type="text/markdown" href="..."> tag
|
|
237
|
+
* 2. VitePress markers → append .md to the URL
|
|
238
|
+
*
|
|
239
|
+
* The llms.txt .md probe is handled separately in tryMarkdownSource
|
|
240
|
+
* as a fallback when no markers are found.
|
|
241
|
+
*
|
|
242
|
+
* @internal Exported for testing.
|
|
243
|
+
*/
|
|
244
|
+
export function discoverMarkdownSource(html, url, ext) {
|
|
245
|
+
// 1. Look for <link rel="alternate" type="text/markdown" href="...">
|
|
246
|
+
const linkMatch = html.match(/<link[^>]+rel=["']alternate["'][^>]+type=["']text\/markdown["'][^>]+href=["']([^"']+)["']/i) ??
|
|
247
|
+
html.match(/<link[^>]+type=["']text\/markdown["'][^>]+rel=["']alternate["'][^>]+href=["']([^"']+)["']/i);
|
|
248
|
+
if (linkMatch?.[1]) {
|
|
249
|
+
try {
|
|
250
|
+
return new URL(linkMatch[1], url).href;
|
|
251
|
+
}
|
|
252
|
+
catch {
|
|
253
|
+
/* ignore malformed URLs */
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
// 2. VitePress detection — serves .md alongside HTML
|
|
257
|
+
if (!ext &&
|
|
258
|
+
(html.includes("__VP_HASH_MAP__") ||
|
|
259
|
+
html.includes("VPContent") ||
|
|
260
|
+
html.includes("vitepress"))) {
|
|
261
|
+
return appendMdExtension(url);
|
|
262
|
+
}
|
|
263
|
+
return null;
|
|
264
|
+
}
|
|
265
|
+
function appendMdExtension(url) {
|
|
266
|
+
return url.endsWith("/") ? `${url.slice(0, -1)}.md` : `${url}.md`;
|
|
267
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { MarkitPluginAPI, PluginDef, PluginFunction } from "./types.js";
|
|
2
|
+
export declare function createPluginAPI(pluginId: string): {
|
|
3
|
+
api: MarkitPluginAPI;
|
|
4
|
+
resolve: () => PluginDef;
|
|
5
|
+
};
|
|
6
|
+
export declare function isPluginFunction(val: any): val is PluginFunction;
|
|
7
|
+
export declare function resolvePluginExport(exported: PluginFunction | PluginDef, pluginId: string): PluginDef;
|