@steipete/summarize 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +121 -0
- package/LICENSE +1 -1
- package/README.md +391 -183
- package/dist/cli.js +1 -1
- package/dist/esm/cache.js +134 -64
- package/dist/esm/cache.js.map +1 -1
- package/dist/esm/cli-main.js +27 -27
- package/dist/esm/cli-main.js.map +1 -1
- package/dist/esm/cli.js +2 -2
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/config.js +396 -126
- package/dist/esm/config.js.map +1 -1
- package/dist/esm/content/asset.js +53 -50
- package/dist/esm/content/asset.js.map +1 -1
- package/dist/esm/content/index.js +1 -1
- package/dist/esm/content/index.js.map +1 -1
- package/dist/esm/costs.js +1 -1
- package/dist/esm/costs.js.map +1 -1
- package/dist/esm/daemon/agent.js +548 -0
- package/dist/esm/daemon/agent.js.map +1 -0
- package/dist/esm/daemon/auto-mode.js +3 -3
- package/dist/esm/daemon/auto-mode.js.map +1 -1
- package/dist/esm/daemon/chat.js +88 -178
- package/dist/esm/daemon/chat.js.map +1 -1
- package/dist/esm/daemon/cli-entrypoint.js +72 -0
- package/dist/esm/daemon/cli-entrypoint.js.map +1 -0
- package/dist/esm/daemon/cli.js +91 -83
- package/dist/esm/daemon/cli.js.map +1 -1
- package/dist/esm/daemon/config.js +15 -15
- package/dist/esm/daemon/config.js.map +1 -1
- package/dist/esm/daemon/constants.js +6 -6
- package/dist/esm/daemon/constants.js.map +1 -1
- package/dist/esm/daemon/env-merge.js.map +1 -1
- package/dist/esm/daemon/env-snapshot.js +36 -28
- package/dist/esm/daemon/env-snapshot.js.map +1 -1
- package/dist/esm/daemon/flow-context.js +86 -32
- package/dist/esm/daemon/flow-context.js.map +1 -1
- package/dist/esm/daemon/launchd.js +119 -47
- package/dist/esm/daemon/launchd.js.map +1 -1
- package/dist/esm/daemon/meta.js +5 -5
- package/dist/esm/daemon/meta.js.map +1 -1
- package/dist/esm/daemon/models.js +54 -31
- package/dist/esm/daemon/models.js.map +1 -1
- package/dist/esm/daemon/process-registry.js +206 -0
- package/dist/esm/daemon/process-registry.js.map +1 -0
- package/dist/esm/daemon/schtasks.js +96 -32
- package/dist/esm/daemon/schtasks.js.map +1 -1
- package/dist/esm/daemon/server.js +832 -158
- package/dist/esm/daemon/server.js.map +1 -1
- package/dist/esm/daemon/summarize-progress.js +11 -11
- package/dist/esm/daemon/summarize-progress.js.map +1 -1
- package/dist/esm/daemon/summarize.js +61 -32
- package/dist/esm/daemon/summarize.js.map +1 -1
- package/dist/esm/daemon/systemd.js +96 -35
- package/dist/esm/daemon/systemd.js.map +1 -1
- package/dist/esm/firecrawl.js +12 -12
- package/dist/esm/firecrawl.js.map +1 -1
- package/dist/esm/flags.js +55 -31
- package/dist/esm/flags.js.map +1 -1
- package/dist/esm/index.js +3 -3
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/language.js +1 -1
- package/dist/esm/language.js.map +1 -1
- package/dist/esm/llm/cli.js +128 -64
- package/dist/esm/llm/cli.js.map +1 -1
- package/dist/esm/llm/errors.js +1 -1
- package/dist/esm/llm/errors.js.map +1 -1
- package/dist/esm/llm/generate-text.js +107 -98
- package/dist/esm/llm/generate-text.js.map +1 -1
- package/dist/esm/llm/google-models.js +17 -17
- package/dist/esm/llm/google-models.js.map +1 -1
- package/dist/esm/llm/html-to-markdown.js +3 -3
- package/dist/esm/llm/html-to-markdown.js.map +1 -1
- package/dist/esm/llm/model-id.js +38 -16
- package/dist/esm/llm/model-id.js.map +1 -1
- package/dist/esm/llm/prompt.js +5 -5
- package/dist/esm/llm/prompt.js.map +1 -1
- package/dist/esm/llm/providers/anthropic.js +33 -33
- package/dist/esm/llm/providers/anthropic.js.map +1 -1
- package/dist/esm/llm/providers/google.js +19 -19
- package/dist/esm/llm/providers/google.js.map +1 -1
- package/dist/esm/llm/providers/models.js +30 -30
- package/dist/esm/llm/providers/models.js.map +1 -1
- package/dist/esm/llm/providers/openai.js +36 -35
- package/dist/esm/llm/providers/openai.js.map +1 -1
- package/dist/esm/llm/providers/shared.js +8 -8
- package/dist/esm/llm/providers/shared.js.map +1 -1
- package/dist/esm/llm/transcript-to-markdown.js +9 -5
- package/dist/esm/llm/transcript-to-markdown.js.map +1 -1
- package/dist/esm/llm/usage.js +18 -18
- package/dist/esm/llm/usage.js.map +1 -1
- package/dist/esm/logging/daemon.js +21 -21
- package/dist/esm/logging/daemon.js.map +1 -1
- package/dist/esm/logging/ring-file.js +5 -5
- package/dist/esm/logging/ring-file.js.map +1 -1
- package/dist/esm/markitdown.js +21 -19
- package/dist/esm/markitdown.js.map +1 -1
- package/dist/esm/media-cache.js +251 -0
- package/dist/esm/media-cache.js.map +1 -0
- package/dist/esm/model-auto.js +175 -106
- package/dist/esm/model-auto.js.map +1 -1
- package/dist/esm/model-spec.js +52 -42
- package/dist/esm/model-spec.js.map +1 -1
- package/dist/esm/pricing/litellm.js +4 -4
- package/dist/esm/pricing/litellm.js.map +1 -1
- package/dist/esm/processes.js +2 -0
- package/dist/esm/processes.js.map +1 -0
- package/dist/esm/prompts/index.js +1 -1
- package/dist/esm/prompts/index.js.map +1 -1
- package/dist/esm/refresh-free.js +81 -81
- package/dist/esm/refresh-free.js.map +1 -1
- package/dist/esm/run/attachments.js +47 -44
- package/dist/esm/run/attachments.js.map +1 -1
- package/dist/esm/run/bird.js +125 -12
- package/dist/esm/run/bird.js.map +1 -1
- package/dist/esm/run/cache-state.js +7 -7
- package/dist/esm/run/cache-state.js.map +1 -1
- package/dist/esm/run/cli-fallback-state.js +45 -0
- package/dist/esm/run/cli-fallback-state.js.map +1 -0
- package/dist/esm/run/cli-preflight.js +40 -22
- package/dist/esm/run/cli-preflight.js.map +1 -1
- package/dist/esm/run/constants.js +12 -12
- package/dist/esm/run/constants.js.map +1 -1
- package/dist/esm/run/cookies/twitter.js +47 -47
- package/dist/esm/run/cookies/twitter.js.map +1 -1
- package/dist/esm/run/env.js +21 -15
- package/dist/esm/run/env.js.map +1 -1
- package/dist/esm/run/fetch-with-timeout.js +4 -4
- package/dist/esm/run/fetch-with-timeout.js.map +1 -1
- package/dist/esm/run/finish-line.js +78 -71
- package/dist/esm/run/finish-line.js.map +1 -1
- package/dist/esm/run/flows/asset/extract.js +70 -0
- package/dist/esm/run/flows/asset/extract.js.map +1 -0
- package/dist/esm/run/flows/asset/input.js +202 -37
- package/dist/esm/run/flows/asset/input.js.map +1 -1
- package/dist/esm/run/flows/asset/media-policy.js +3 -0
- package/dist/esm/run/flows/asset/media-policy.js.map +1 -0
- package/dist/esm/run/flows/asset/media.js +233 -0
- package/dist/esm/run/flows/asset/media.js.map +1 -0
- package/dist/esm/run/flows/asset/output.js +98 -0
- package/dist/esm/run/flows/asset/output.js.map +1 -0
- package/dist/esm/run/flows/asset/preprocess.js +79 -44
- package/dist/esm/run/flows/asset/preprocess.js.map +1 -1
- package/dist/esm/run/flows/asset/summary.js +306 -89
- package/dist/esm/run/flows/asset/summary.js.map +1 -1
- package/dist/esm/run/flows/url/extract.js +31 -31
- package/dist/esm/run/flows/url/extract.js.map +1 -1
- package/dist/esm/run/flows/url/flow.js +388 -82
- package/dist/esm/run/flows/url/flow.js.map +1 -1
- package/dist/esm/run/flows/url/markdown.js +61 -56
- package/dist/esm/run/flows/url/markdown.js.map +1 -1
- package/dist/esm/run/flows/url/slides-output.js +487 -0
- package/dist/esm/run/flows/url/slides-output.js.map +1 -0
- package/dist/esm/run/flows/url/slides-text.js +628 -0
- package/dist/esm/run/flows/url/slides-text.js.map +1 -0
- package/dist/esm/run/flows/url/summary.js +493 -152
- package/dist/esm/run/flows/url/summary.js.map +1 -1
- package/dist/esm/run/format.js +10 -10
- package/dist/esm/run/format.js.map +1 -1
- package/dist/esm/run/help.js +179 -84
- package/dist/esm/run/help.js.map +1 -1
- package/dist/esm/run/logging.js +20 -12
- package/dist/esm/run/logging.js.map +1 -1
- package/dist/esm/run/markdown.js +12 -12
- package/dist/esm/run/markdown.js.map +1 -1
- package/dist/esm/run/media-cache-state.js +33 -0
- package/dist/esm/run/media-cache-state.js.map +1 -0
- package/dist/esm/run/model-attempts.js.map +1 -1
- package/dist/esm/run/openrouter.js +11 -11
- package/dist/esm/run/openrouter.js.map +1 -1
- package/dist/esm/run/progress.js +19 -1
- package/dist/esm/run/progress.js.map +1 -1
- package/dist/esm/run/run-config.js +16 -16
- package/dist/esm/run/run-config.js.map +1 -1
- package/dist/esm/run/run-context.js +2 -2
- package/dist/esm/run/run-context.js.map +1 -1
- package/dist/esm/run/run-env.js +55 -54
- package/dist/esm/run/run-env.js.map +1 -1
- package/dist/esm/run/run-input.js +3 -3
- package/dist/esm/run/run-input.js.map +1 -1
- package/dist/esm/run/run-metrics.js +16 -16
- package/dist/esm/run/run-metrics.js.map +1 -1
- package/dist/esm/run/run-models.js +28 -23
- package/dist/esm/run/run-models.js.map +1 -1
- package/dist/esm/run/run-output.js +3 -3
- package/dist/esm/run/run-output.js.map +1 -1
- package/dist/esm/run/run-settings.js +108 -21
- package/dist/esm/run/run-settings.js.map +1 -1
- package/dist/esm/run/run-stream.js +4 -4
- package/dist/esm/run/run-stream.js.map +1 -1
- package/dist/esm/run/runner.js +327 -100
- package/dist/esm/run/runner.js.map +1 -1
- package/dist/esm/run/slides-cli.js +226 -0
- package/dist/esm/run/slides-cli.js.map +1 -0
- package/dist/esm/run/slides-render.js +163 -0
- package/dist/esm/run/slides-render.js.map +1 -0
- package/dist/esm/run/stdin-temp-file.js +77 -0
- package/dist/esm/run/stdin-temp-file.js.map +1 -0
- package/dist/esm/run/stream-output.js +17 -10
- package/dist/esm/run/stream-output.js.map +1 -1
- package/dist/esm/run/streaming.js +16 -16
- package/dist/esm/run/streaming.js.map +1 -1
- package/dist/esm/run/summary-engine.js +89 -57
- package/dist/esm/run/summary-engine.js.map +1 -1
- package/dist/esm/run/summary-llm.js +3 -3
- package/dist/esm/run/summary-llm.js.map +1 -1
- package/dist/esm/run/terminal.js +4 -4
- package/dist/esm/run/terminal.js.map +1 -1
- package/dist/esm/run/tips.js +2 -2
- package/dist/esm/run/tips.js.map +1 -1
- package/dist/esm/run/transcriber-cli.js +148 -0
- package/dist/esm/run/transcriber-cli.js.map +1 -0
- package/dist/esm/run.js +1 -1
- package/dist/esm/run.js.map +1 -1
- package/dist/esm/shared/contracts.js +1 -1
- package/dist/esm/shared/contracts.js.map +1 -1
- package/dist/esm/shared/sse-events.js +16 -12
- package/dist/esm/shared/sse-events.js.map +1 -1
- package/dist/esm/shared/streaming-merge.js +3 -3
- package/dist/esm/shared/streaming-merge.js.map +1 -1
- package/dist/esm/slides/extract.js +1951 -0
- package/dist/esm/slides/extract.js.map +1 -0
- package/dist/esm/slides/index.js +4 -0
- package/dist/esm/slides/index.js.map +1 -0
- package/dist/esm/slides/settings.js +73 -0
- package/dist/esm/slides/settings.js.map +1 -0
- package/dist/esm/slides/store.js +111 -0
- package/dist/esm/slides/store.js.map +1 -0
- package/dist/esm/slides/types.js +2 -0
- package/dist/esm/slides/types.js.map +1 -0
- package/dist/esm/tty/format.js +13 -13
- package/dist/esm/tty/format.js.map +1 -1
- package/dist/esm/tty/osc-progress.js +22 -2
- package/dist/esm/tty/osc-progress.js.map +1 -1
- package/dist/esm/tty/progress/fetch-html.js +20 -16
- package/dist/esm/tty/progress/fetch-html.js.map +1 -1
- package/dist/esm/tty/progress/transcript.js +127 -68
- package/dist/esm/tty/progress/transcript.js.map +1 -1
- package/dist/esm/tty/spinner.js +21 -10
- package/dist/esm/tty/spinner.js.map +1 -1
- package/dist/esm/tty/theme.js +189 -0
- package/dist/esm/tty/theme.js.map +1 -0
- package/dist/esm/tty/website-progress.js +38 -34
- package/dist/esm/tty/website-progress.js.map +1 -1
- package/dist/esm/version.js +29 -29
- package/dist/esm/version.js.map +1 -1
- package/dist/types/cache.d.ts +19 -7
- package/dist/types/config.d.ts +71 -6
- package/dist/types/content/asset.d.ts +8 -6
- package/dist/types/content/index.d.ts +1 -1
- package/dist/types/costs.d.ts +3 -3
- package/dist/types/daemon/agent.d.ts +25 -0
- package/dist/types/daemon/auto-mode.d.ts +3 -3
- package/dist/types/daemon/chat.d.ts +10 -18
- package/dist/types/daemon/cli-entrypoint.d.ts +2 -0
- package/dist/types/daemon/config.d.ts +2 -2
- package/dist/types/daemon/env-merge.d.ts +1 -1
- package/dist/types/daemon/env-snapshot.d.ts +1 -1
- package/dist/types/daemon/flow-context.d.ts +24 -4
- package/dist/types/daemon/launchd.d.ts +12 -0
- package/dist/types/daemon/models.d.ts +6 -2
- package/dist/types/daemon/process-registry.d.ts +73 -0
- package/dist/types/daemon/schtasks.d.ts +4 -0
- package/dist/types/daemon/server.d.ts +2 -2
- package/dist/types/daemon/summarize-progress.d.ts +1 -1
- package/dist/types/daemon/summarize.d.ts +38 -7
- package/dist/types/daemon/systemd.d.ts +4 -0
- package/dist/types/firecrawl.d.ts +1 -1
- package/dist/types/flags.d.ts +12 -11
- package/dist/types/index.d.ts +4 -4
- package/dist/types/language.d.ts +1 -1
- package/dist/types/llm/attachments.d.ts +1 -1
- package/dist/types/llm/cli.d.ts +3 -3
- package/dist/types/llm/generate-text.d.ts +7 -7
- package/dist/types/llm/html-to-markdown.d.ts +3 -3
- package/dist/types/llm/model-id.d.ts +1 -1
- package/dist/types/llm/prompt.d.ts +2 -2
- package/dist/types/llm/providers/anthropic.d.ts +3 -3
- package/dist/types/llm/providers/google.d.ts +3 -3
- package/dist/types/llm/providers/models.d.ts +2 -2
- package/dist/types/llm/providers/openai.d.ts +4 -4
- package/dist/types/llm/providers/shared.d.ts +2 -2
- package/dist/types/llm/transcript-to-markdown.d.ts +4 -2
- package/dist/types/llm/usage.d.ts +1 -1
- package/dist/types/logging/daemon.d.ts +4 -4
- package/dist/types/markitdown.d.ts +1 -1
- package/dist/types/media-cache.d.ts +22 -0
- package/dist/types/model-auto.d.ts +14 -4
- package/dist/types/model-spec.d.ts +10 -10
- package/dist/types/pricing/litellm.d.ts +1 -1
- package/dist/types/processes.d.ts +1 -0
- package/dist/types/prompts/index.d.ts +1 -1
- package/dist/types/run/attachments.d.ts +7 -7
- package/dist/types/run/bird.d.ts +7 -0
- package/dist/types/run/cache-state.d.ts +2 -2
- package/dist/types/run/cli-fallback-state.d.ts +6 -0
- package/dist/types/run/constants.d.ts +1 -1
- package/dist/types/run/cookies/twitter.d.ts +1 -1
- package/dist/types/run/env.d.ts +1 -1
- package/dist/types/run/finish-line.d.ts +7 -6
- package/dist/types/run/flows/asset/extract.d.ts +18 -0
- package/dist/types/run/flows/asset/input.d.ts +19 -3
- package/dist/types/run/flows/asset/media-policy.d.ts +2 -0
- package/dist/types/run/flows/asset/media.d.ts +21 -0
- package/dist/types/run/flows/asset/output.d.ts +42 -0
- package/dist/types/run/flows/asset/preprocess.d.ts +23 -17
- package/dist/types/run/flows/asset/summary.d.ts +24 -16
- package/dist/types/run/flows/url/extract.d.ts +3 -2
- package/dist/types/run/flows/url/flow.d.ts +1 -1
- package/dist/types/run/flows/url/markdown.d.ts +6 -6
- package/dist/types/run/flows/url/slides-output.d.ts +66 -0
- package/dist/types/run/flows/url/slides-text.d.ts +87 -0
- package/dist/types/run/flows/url/summary.d.ts +18 -10
- package/dist/types/run/flows/url/types.d.ts +52 -21
- package/dist/types/run/format.d.ts +3 -3
- package/dist/types/run/help.d.ts +4 -1
- package/dist/types/run/logging.d.ts +3 -2
- package/dist/types/run/media-cache-state.d.ts +7 -0
- package/dist/types/run/model-attempts.d.ts +1 -1
- package/dist/types/run/progress.d.ts +2 -1
- package/dist/types/run/run-config.d.ts +4 -4
- package/dist/types/run/run-context.d.ts +3 -1
- package/dist/types/run/run-env.d.ts +3 -1
- package/dist/types/run/run-input.d.ts +2 -2
- package/dist/types/run/run-metrics.d.ts +3 -3
- package/dist/types/run/run-models.d.ts +3 -2
- package/dist/types/run/run-output.d.ts +1 -1
- package/dist/types/run/run-settings.d.ts +20 -5
- package/dist/types/run/run-stream.d.ts +2 -2
- package/dist/types/run/runner.d.ts +3 -2
- package/dist/types/run/slides-cli.d.ts +9 -0
- package/dist/types/run/slides-render.d.ts +30 -0
- package/dist/types/run/stdin-temp-file.d.ts +9 -0
- package/dist/types/run/stream-output.d.ts +3 -2
- package/dist/types/run/streaming.d.ts +4 -4
- package/dist/types/run/summary-engine.d.ts +22 -12
- package/dist/types/run/summary-llm.d.ts +5 -5
- package/dist/types/run/transcriber-cli.d.ts +8 -0
- package/dist/types/run/types.d.ts +4 -4
- package/dist/types/run.d.ts +1 -1
- package/dist/types/shared/contracts.d.ts +2 -2
- package/dist/types/shared/sse-events.d.ts +26 -6
- package/dist/types/slides/extract.d.ts +43 -0
- package/dist/types/slides/index.d.ts +5 -0
- package/dist/types/slides/settings.d.ts +20 -0
- package/dist/types/slides/store.d.ts +15 -0
- package/dist/types/slides/types.d.ts +40 -0
- package/dist/types/tty/osc-progress.d.ts +5 -5
- package/dist/types/tty/progress/fetch-html.d.ts +5 -3
- package/dist/types/tty/progress/transcript.d.ts +5 -3
- package/dist/types/tty/spinner.d.ts +3 -1
- package/dist/types/tty/theme.d.ts +44 -0
- package/dist/types/tty/website-progress.d.ts +5 -3
- package/dist/types/version.d.ts +1 -1
- package/docs/README.md +1 -1
- package/docs/_config.yml +26 -0
- package/docs/_layouts/default.html +60 -0
- package/docs/agent.md +367 -0
- package/docs/assets/site.css +748 -0
- package/docs/assets/site.js +72 -0
- package/docs/assets/summarize-cli.png +0 -0
- package/docs/assets/summarize-extension.png +0 -0
- package/docs/assets/youtube-slides.png +0 -0
- package/docs/cache.md +29 -3
- package/docs/chrome-extension.md +72 -16
- package/docs/cli.md +59 -13
- package/docs/config.md +109 -12
- package/docs/extract-only.md +10 -0
- package/docs/index.html +224 -0
- package/docs/index.md +25 -0
- package/docs/llm.md +18 -5
- package/docs/manual-tests.md +2 -0
- package/docs/media.md +6 -2
- package/docs/model-auto.md +3 -2
- package/docs/nvidia-onnx-transcription.md +55 -0
- package/docs/openai.md +1 -1
- package/docs/releasing.md +3 -0
- package/docs/site/404.html +4 -1
- package/docs/site/assets/site.css +399 -228
- package/docs/site/assets/site.js +46 -46
- package/docs/site/assets/summarize-cli.png +0 -0
- package/docs/site/assets/summarize-extension.png +0 -0
- package/docs/site/docs/chrome-extension.html +101 -0
- package/docs/site/docs/config.html +30 -8
- package/docs/site/docs/extract-only.html +17 -4
- package/docs/site/docs/firecrawl.html +13 -3
- package/docs/site/docs/index.html +40 -6
- package/docs/site/docs/llm.html +20 -5
- package/docs/site/docs/openai.html +19 -5
- package/docs/site/docs/website.html +30 -9
- package/docs/site/docs/youtube.html +13 -3
- package/docs/site/index.html +168 -85
- package/docs/slides.md +82 -0
- package/docs/smoketest.md +29 -20
- package/docs/timestamps.md +124 -0
- package/docs/website.md +13 -0
- package/docs/youtube.md +20 -0
- package/package.json +57 -48
|
@@ -1,37 +1,46 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
1
|
+
import * as urlUtils from "@steipete/summarize-core/content/url";
|
|
2
|
+
import { buildExtractCacheKey, buildSlidesCacheKey } from "../../../cache.js";
|
|
3
|
+
import { loadRemoteAsset } from "../../../content/asset.js";
|
|
4
|
+
import { createLinkPreviewClient, } from "../../../content/index.js";
|
|
5
|
+
import { createFirecrawlScraper } from "../../../firecrawl.js";
|
|
6
|
+
import { extractSlidesForSource, resolveSlideSource, validateSlidesCache, } from "../../../slides/index.js";
|
|
7
|
+
import { createOscProgressController } from "../../../tty/osc-progress.js";
|
|
8
|
+
import { startSpinner } from "../../../tty/spinner.js";
|
|
9
|
+
import { createThemeRenderer, resolveThemeNameFromSources, resolveTrueColor, } from "../../../tty/theme.js";
|
|
10
|
+
import { createWebsiteProgress } from "../../../tty/website-progress.js";
|
|
11
|
+
import { assertAssetMediaTypeSupported } from "../../attachments.js";
|
|
12
|
+
import { readTweetWithBird } from "../../bird.js";
|
|
13
|
+
import { UVX_TIP } from "../../constants.js";
|
|
14
|
+
import { resolveTwitterCookies } from "../../cookies/twitter.js";
|
|
15
|
+
import { hasBirdCli, hasUvxCli } from "../../env.js";
|
|
16
|
+
import { estimateWhisperTranscriptionCostUsd, formatOptionalNumber, formatOptionalString, formatUSD, } from "../../format.js";
|
|
17
|
+
import { writeVerbose } from "../../logging.js";
|
|
18
|
+
import { deriveExtractionUi, fetchLinkContentWithBirdTip, logExtractionDiagnostics, } from "./extract.js";
|
|
19
|
+
import { createMarkdownConverters } from "./markdown.js";
|
|
20
|
+
import { createSlidesTerminalOutput } from "./slides-output.js";
|
|
21
|
+
import { buildUrlPrompt, outputExtractedUrl, summarizeExtractedUrl } from "./summary.js";
|
|
18
22
|
export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
19
23
|
if (!url) {
|
|
20
|
-
throw new Error(
|
|
24
|
+
throw new Error("Only HTTP and HTTPS URLs can be summarized");
|
|
21
25
|
}
|
|
22
26
|
const { io, flags, model, cache: cacheState, hooks } = ctx;
|
|
27
|
+
const theme = createThemeRenderer({
|
|
28
|
+
themeName: resolveThemeNameFromSources({ env: io.envForRun.SUMMARIZE_THEME }),
|
|
29
|
+
enabled: flags.verboseColor,
|
|
30
|
+
trueColor: resolveTrueColor(io.envForRun),
|
|
31
|
+
});
|
|
23
32
|
const markdown = createMarkdownConverters(ctx, { isYoutubeUrl });
|
|
24
|
-
if (flags.firecrawlMode ===
|
|
25
|
-
throw new Error(
|
|
33
|
+
if (flags.firecrawlMode === "always" && !model.apiStatus.firecrawlConfigured) {
|
|
34
|
+
throw new Error("--firecrawl always requires FIRECRAWL_API_KEY");
|
|
26
35
|
}
|
|
27
|
-
writeVerbose(io.stderr, flags.verbose, `config url=${url} timeoutMs=${flags.timeoutMs} youtube=${flags.youtubeMode} firecrawl=${flags.firecrawlMode} length=${flags.lengthArg.kind ===
|
|
36
|
+
writeVerbose(io.stderr, flags.verbose, `config url=${url} timeoutMs=${flags.timeoutMs} youtube=${flags.youtubeMode} firecrawl=${flags.firecrawlMode} length=${flags.lengthArg.kind === "preset"
|
|
28
37
|
? flags.lengthArg.preset
|
|
29
|
-
: `${flags.lengthArg.maxCharacters} chars`} maxOutputTokens=${formatOptionalNumber(flags.maxOutputTokensArg)} retries=${flags.retries} json=${flags.json} extract=${flags.extractMode} format=${flags.format} preprocess=${flags.preprocessMode} markdownMode=${flags.markdownMode} model=${model.requestedModelLabel} videoMode=${flags.videoMode} stream=${flags.streamingEnabled ?
|
|
30
|
-
writeVerbose(io.stderr, flags.verbose, `configFile path=${formatOptionalString(flags.configPath)} model=${formatOptionalString(flags.configModelLabel)}`, flags.verboseColor);
|
|
31
|
-
writeVerbose(io.stderr, flags.verbose, `env xaiKey=${Boolean(model.apiStatus.xaiApiKey)} openaiKey=${Boolean(model.apiStatus.apiKey)} zaiKey=${Boolean(model.apiStatus.zaiApiKey)} googleKey=${model.apiStatus.googleConfigured} anthropicKey=${model.apiStatus.anthropicConfigured} openrouterKey=${model.apiStatus.openrouterConfigured} apifyToken=${Boolean(model.apiStatus.apifyToken)} firecrawlKey=${model.apiStatus.firecrawlConfigured}`, flags.verboseColor);
|
|
32
|
-
writeVerbose(io.stderr, flags.verbose, `markdown htmlRequested=${markdown.markdownRequested} transcriptRequested=${markdown.transcriptMarkdownRequested} provider=${markdown.markdownProvider}`, flags.verboseColor);
|
|
38
|
+
: `${flags.lengthArg.maxCharacters} chars`} maxOutputTokens=${formatOptionalNumber(flags.maxOutputTokensArg)} retries=${flags.retries} json=${flags.json} extract=${flags.extractMode} format=${flags.format} preprocess=${flags.preprocessMode} markdownMode=${flags.markdownMode} model=${model.requestedModelLabel} videoMode=${flags.videoMode} timestamps=${flags.transcriptTimestamps ? "on" : "off"} stream=${flags.streamingEnabled ? "on" : "off"} plain=${flags.plain}`, flags.verboseColor, io.envForRun);
|
|
39
|
+
writeVerbose(io.stderr, flags.verbose, `configFile path=${formatOptionalString(flags.configPath)} model=${formatOptionalString(flags.configModelLabel)}`, flags.verboseColor, io.envForRun);
|
|
40
|
+
writeVerbose(io.stderr, flags.verbose, `env xaiKey=${Boolean(model.apiStatus.xaiApiKey)} openaiKey=${Boolean(model.apiStatus.apiKey)} zaiKey=${Boolean(model.apiStatus.zaiApiKey)} googleKey=${model.apiStatus.googleConfigured} anthropicKey=${model.apiStatus.anthropicConfigured} openrouterKey=${model.apiStatus.openrouterConfigured} apifyToken=${Boolean(model.apiStatus.apifyToken)} firecrawlKey=${model.apiStatus.firecrawlConfigured}`, flags.verboseColor, io.envForRun);
|
|
41
|
+
writeVerbose(io.stderr, flags.verbose, `markdown htmlRequested=${markdown.markdownRequested} transcriptRequested=${markdown.transcriptMarkdownRequested} provider=${markdown.markdownProvider}`, flags.verboseColor, io.envForRun);
|
|
33
42
|
const firecrawlApiKey = model.apiStatus.firecrawlApiKey;
|
|
34
|
-
const scrapeWithFirecrawl = model.apiStatus.firecrawlConfigured && flags.firecrawlMode !==
|
|
43
|
+
const scrapeWithFirecrawl = model.apiStatus.firecrawlConfigured && flags.firecrawlMode !== "off" && firecrawlApiKey
|
|
35
44
|
? createFirecrawlScraper({
|
|
36
45
|
apiKey: firecrawlApiKey,
|
|
37
46
|
fetchImpl: io.fetch,
|
|
@@ -40,31 +49,82 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
40
49
|
const readTweetWithBirdClient = hasBirdCli(io.env)
|
|
41
50
|
? ({ url, timeoutMs }) => readTweetWithBird({ url, timeoutMs, env: io.env })
|
|
42
51
|
: null;
|
|
43
|
-
writeVerbose(io.stderr, flags.verbose,
|
|
52
|
+
writeVerbose(io.stderr, flags.verbose, "extract start", flags.verboseColor, io.envForRun);
|
|
44
53
|
const oscProgress = createOscProgressController({
|
|
45
|
-
label:
|
|
54
|
+
label: "Fetching website",
|
|
46
55
|
env: io.env,
|
|
47
56
|
isTty: flags.progressEnabled,
|
|
48
57
|
write: (data) => io.stderr.write(data),
|
|
49
58
|
});
|
|
50
|
-
oscProgress.setIndeterminate(
|
|
59
|
+
oscProgress.setIndeterminate("Fetching website");
|
|
51
60
|
const spinner = startSpinner({
|
|
52
|
-
text:
|
|
61
|
+
text: `${theme.label("Fetching website")}${theme.dim(" (connecting)…")}`,
|
|
53
62
|
enabled: flags.progressEnabled,
|
|
54
63
|
stream: io.stderr,
|
|
64
|
+
color: theme.palette.spinner,
|
|
55
65
|
});
|
|
66
|
+
const styleLabel = (text) => theme.label(text);
|
|
67
|
+
const styleDim = (text) => theme.dim(text);
|
|
68
|
+
const renderStatus = (label, detail = "…") => `${styleLabel(label)}${styleDim(detail)}`;
|
|
69
|
+
const renderStatusWithMeta = (label, meta, suffix = "…") => `${styleLabel(label)} ${meta}${styleDim(suffix)}`;
|
|
70
|
+
const renderStatusFromText = (text) => {
|
|
71
|
+
const match = text.match(/^([^:]+):(.*)$/);
|
|
72
|
+
if (!match)
|
|
73
|
+
return styleLabel(text);
|
|
74
|
+
return `${styleLabel(match[1])}${styleDim(`:${match[2]}`)}`;
|
|
75
|
+
};
|
|
76
|
+
const handleSignal = () => {
|
|
77
|
+
try {
|
|
78
|
+
spinner.stopAndClear();
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
// ignore
|
|
82
|
+
}
|
|
83
|
+
oscProgress.clear();
|
|
84
|
+
};
|
|
85
|
+
const handleSigint = () => {
|
|
86
|
+
handleSignal();
|
|
87
|
+
process.exit(130);
|
|
88
|
+
};
|
|
89
|
+
const handleSigterm = () => {
|
|
90
|
+
handleSignal();
|
|
91
|
+
process.exit(143);
|
|
92
|
+
};
|
|
93
|
+
if (flags.progressEnabled) {
|
|
94
|
+
process.once("SIGINT", handleSigint);
|
|
95
|
+
process.once("SIGTERM", handleSigterm);
|
|
96
|
+
}
|
|
97
|
+
if (!hooks.onSlidesProgress && flags.progressEnabled) {
|
|
98
|
+
hooks.onSlidesProgress = (text) => {
|
|
99
|
+
const match = text.match(/(\d{1,3})%/);
|
|
100
|
+
const percent = match ? Number(match[1]) : null;
|
|
101
|
+
spinner.setText(renderStatusFromText(text));
|
|
102
|
+
if (Number.isFinite(percent) && percent !== null) {
|
|
103
|
+
oscProgress.setPercent("Slides", Math.max(0, Math.min(100, percent)));
|
|
104
|
+
}
|
|
105
|
+
else {
|
|
106
|
+
oscProgress.setIndeterminate("Slides");
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
}
|
|
56
110
|
const websiteProgress = createWebsiteProgress({
|
|
57
111
|
enabled: flags.progressEnabled,
|
|
58
112
|
spinner,
|
|
59
113
|
oscProgress,
|
|
114
|
+
theme,
|
|
60
115
|
});
|
|
61
|
-
const cacheStore = cacheState.mode ===
|
|
116
|
+
const cacheStore = cacheState.mode === "default" ? cacheState.store : null;
|
|
62
117
|
const transcriptCache = cacheStore ? cacheStore.transcriptCache : null;
|
|
63
118
|
const client = createLinkPreviewClient({
|
|
119
|
+
env: io.envForRun,
|
|
64
120
|
apifyApiToken: model.apiStatus.apifyToken,
|
|
65
121
|
ytDlpPath: model.apiStatus.ytDlpPath,
|
|
66
|
-
|
|
67
|
-
|
|
122
|
+
transcription: {
|
|
123
|
+
env: io.envForRun,
|
|
124
|
+
falApiKey: model.apiStatus.falApiKey,
|
|
125
|
+
groqApiKey: model.apiStatus.groqApiKey,
|
|
126
|
+
openaiApiKey: model.apiStatus.openaiTranscriptionKey,
|
|
127
|
+
},
|
|
68
128
|
scrapeWithFirecrawl,
|
|
69
129
|
convertHtmlToMarkdown: markdown.convertHtmlToMarkdown,
|
|
70
130
|
readTweetWithBird: readTweetWithBirdClient,
|
|
@@ -78,6 +138,7 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
78
138
|
},
|
|
79
139
|
fetch: io.fetch,
|
|
80
140
|
transcriptCache,
|
|
141
|
+
mediaCache: ctx.mediaCache ?? null,
|
|
81
142
|
onProgress: websiteProgress || hooks.onLinkPreviewProgress
|
|
82
143
|
? (event) => {
|
|
83
144
|
websiteProgress?.onProgress(event);
|
|
@@ -94,26 +155,28 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
94
155
|
spinner.stopAndClear();
|
|
95
156
|
oscProgress.clear();
|
|
96
157
|
};
|
|
97
|
-
const
|
|
98
|
-
|
|
158
|
+
const pauseProgressLine = () => {
|
|
159
|
+
spinner.pause();
|
|
160
|
+
return () => spinner.resume();
|
|
99
161
|
};
|
|
100
|
-
hooks.setClearProgressBeforeStdout(
|
|
162
|
+
hooks.setClearProgressBeforeStdout(pauseProgressLine);
|
|
101
163
|
try {
|
|
102
164
|
const buildFetchOptions = () => ({
|
|
103
165
|
timeoutMs: flags.timeoutMs,
|
|
104
|
-
maxCharacters: typeof flags.maxExtractCharacters ===
|
|
166
|
+
maxCharacters: typeof flags.maxExtractCharacters === "number" && flags.maxExtractCharacters > 0
|
|
105
167
|
? flags.maxExtractCharacters
|
|
106
168
|
: undefined,
|
|
107
169
|
youtubeTranscript: flags.youtubeMode,
|
|
108
|
-
mediaTranscript: flags.videoMode ===
|
|
170
|
+
mediaTranscript: flags.videoMode === "transcript" ? "prefer" : "auto",
|
|
171
|
+
transcriptTimestamps: flags.transcriptTimestamps,
|
|
109
172
|
firecrawl: flags.firecrawlMode,
|
|
110
|
-
format: markdown.markdownRequested ?
|
|
173
|
+
format: markdown.markdownRequested ? "markdown" : "text",
|
|
111
174
|
markdownMode: markdown.markdownRequested ? markdown.effectiveMarkdownMode : undefined,
|
|
112
175
|
cacheMode: cacheState.mode,
|
|
113
176
|
});
|
|
114
|
-
const fetchWithCache = async (targetUrl) => {
|
|
177
|
+
const fetchWithCache = async (targetUrl, { bypassExtractCache = false, } = {}) => {
|
|
115
178
|
const options = buildFetchOptions();
|
|
116
|
-
const cacheKey = cacheStore && cacheState.mode ===
|
|
179
|
+
const cacheKey = cacheStore && cacheState.mode === "default"
|
|
117
180
|
? buildExtractCacheKey({
|
|
118
181
|
url: targetUrl,
|
|
119
182
|
options: {
|
|
@@ -122,44 +185,259 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
122
185
|
firecrawl: options.firecrawl,
|
|
123
186
|
format: options.format,
|
|
124
187
|
markdownMode: options.markdownMode ?? null,
|
|
125
|
-
|
|
188
|
+
transcriptTimestamps: options.transcriptTimestamps ?? false,
|
|
189
|
+
...(typeof options.maxCharacters === "number"
|
|
126
190
|
? { maxCharacters: options.maxCharacters }
|
|
127
191
|
: {}),
|
|
128
192
|
},
|
|
129
193
|
})
|
|
130
194
|
: null;
|
|
131
|
-
if (cacheKey && cacheStore) {
|
|
132
|
-
const cached = cacheStore.getJson(
|
|
195
|
+
if (!bypassExtractCache && cacheKey && cacheStore) {
|
|
196
|
+
const cached = cacheStore.getJson("extract", cacheKey);
|
|
133
197
|
if (cached) {
|
|
134
|
-
writeVerbose(io.stderr, flags.verbose,
|
|
198
|
+
writeVerbose(io.stderr, flags.verbose, "cache hit extract", flags.verboseColor, io.envForRun);
|
|
135
199
|
return cached;
|
|
136
200
|
}
|
|
137
|
-
writeVerbose(io.stderr, flags.verbose,
|
|
201
|
+
writeVerbose(io.stderr, flags.verbose, "cache miss extract", flags.verboseColor, io.envForRun);
|
|
138
202
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
203
|
+
try {
|
|
204
|
+
const extracted = await fetchLinkContentWithBirdTip({
|
|
205
|
+
client,
|
|
206
|
+
url: targetUrl,
|
|
207
|
+
options,
|
|
208
|
+
env: io.env,
|
|
209
|
+
});
|
|
210
|
+
if (cacheKey && cacheStore) {
|
|
211
|
+
cacheStore.setJson("extract", cacheKey, extracted, cacheState.ttlMs);
|
|
212
|
+
writeVerbose(io.stderr, flags.verbose, "cache write extract", flags.verboseColor, io.envForRun);
|
|
213
|
+
}
|
|
214
|
+
return extracted;
|
|
215
|
+
}
|
|
216
|
+
catch (err) {
|
|
217
|
+
const preferUrlMode = typeof urlUtils.shouldPreferUrlMode === "function"
|
|
218
|
+
? urlUtils.shouldPreferUrlMode(targetUrl)
|
|
219
|
+
: false;
|
|
220
|
+
const isTwitter = urlUtils.isTwitterStatusUrl?.(targetUrl) ?? false;
|
|
221
|
+
if (!preferUrlMode || isTwitter)
|
|
222
|
+
throw err;
|
|
223
|
+
// Fallback: skip HTML fetch and proceed with URL-only extraction (YouTube/direct media).
|
|
224
|
+
writeVerbose(io.stderr, flags.verbose, `extract fallback url-only (${err.message ?? String(err)})`, flags.verboseColor, io.envForRun);
|
|
225
|
+
return {
|
|
226
|
+
content: "",
|
|
227
|
+
title: null,
|
|
228
|
+
description: null,
|
|
229
|
+
url: targetUrl,
|
|
230
|
+
siteName: null,
|
|
231
|
+
wordCount: 0,
|
|
232
|
+
totalCharacters: 0,
|
|
233
|
+
truncated: false,
|
|
234
|
+
mediaDurationSeconds: null,
|
|
235
|
+
video: null,
|
|
236
|
+
isVideoOnly: true,
|
|
237
|
+
transcriptSource: null,
|
|
238
|
+
transcriptCharacters: null,
|
|
239
|
+
transcriptWordCount: null,
|
|
240
|
+
transcriptLines: null,
|
|
241
|
+
transcriptMetadata: null,
|
|
242
|
+
transcriptSegments: null,
|
|
243
|
+
transcriptTimedText: null,
|
|
244
|
+
transcriptionProvider: null,
|
|
245
|
+
diagnostics: {
|
|
246
|
+
strategy: "html",
|
|
247
|
+
firecrawl: {
|
|
248
|
+
attempted: false,
|
|
249
|
+
used: false,
|
|
250
|
+
cacheMode: cacheState.mode,
|
|
251
|
+
cacheStatus: "bypassed",
|
|
252
|
+
notes: "skipped (url-only fallback)",
|
|
253
|
+
},
|
|
254
|
+
markdown: {
|
|
255
|
+
requested: false,
|
|
256
|
+
used: false,
|
|
257
|
+
provider: null,
|
|
258
|
+
notes: "skipped (url fallback)",
|
|
259
|
+
},
|
|
260
|
+
transcript: {
|
|
261
|
+
cacheMode: cacheState.mode,
|
|
262
|
+
cacheStatus: "unknown",
|
|
263
|
+
textProvided: false,
|
|
264
|
+
provider: null,
|
|
265
|
+
attemptedProviders: [],
|
|
266
|
+
},
|
|
267
|
+
},
|
|
268
|
+
};
|
|
148
269
|
}
|
|
149
|
-
return extracted;
|
|
150
270
|
};
|
|
151
271
|
let extracted = await fetchWithCache(url);
|
|
272
|
+
if (flags.slides && !resolveSlideSource({ url, extracted })) {
|
|
273
|
+
const isTwitter = urlUtils.isTwitterStatusUrl?.(url) ?? false;
|
|
274
|
+
if (isTwitter) {
|
|
275
|
+
const refreshed = await fetchWithCache(url, { bypassExtractCache: true });
|
|
276
|
+
if (resolveSlideSource({ url, extracted: refreshed })) {
|
|
277
|
+
writeVerbose(io.stderr, flags.verbose, "extract refresh for slides", flags.verboseColor, io.envForRun);
|
|
278
|
+
extracted = refreshed;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
152
282
|
let extractionUi = deriveExtractionUi(extracted);
|
|
283
|
+
let slidesExtracted = null;
|
|
284
|
+
let slidesDone = false;
|
|
285
|
+
let slidesTimelineResolved = false;
|
|
286
|
+
let resolveSlidesTimeline = null;
|
|
287
|
+
const slidesTimelinePromise = flags.slides
|
|
288
|
+
? new Promise((resolve) => {
|
|
289
|
+
resolveSlidesTimeline = resolve;
|
|
290
|
+
})
|
|
291
|
+
: null;
|
|
292
|
+
const resolveTimeline = (value) => {
|
|
293
|
+
if (slidesTimelineResolved)
|
|
294
|
+
return;
|
|
295
|
+
slidesTimelineResolved = true;
|
|
296
|
+
resolveSlidesTimeline?.(value);
|
|
297
|
+
};
|
|
298
|
+
const slidesOutputEnabled = Boolean(flags.slides) && flags.slidesOutput !== false && !flags.json && !flags.extractMode;
|
|
299
|
+
const slidesOutput = createSlidesTerminalOutput({
|
|
300
|
+
io,
|
|
301
|
+
flags: { plain: flags.plain, lengthArg: flags.lengthArg, slidesDebug: flags.slidesDebug },
|
|
302
|
+
extracted,
|
|
303
|
+
slides: null,
|
|
304
|
+
enabled: slidesOutputEnabled,
|
|
305
|
+
outputMode: "delta",
|
|
306
|
+
clearProgressForStdout: hooks.clearProgressForStdout,
|
|
307
|
+
restoreProgressAfterStdout: hooks.restoreProgressAfterStdout ?? null,
|
|
308
|
+
onProgressText: flags.progressEnabled
|
|
309
|
+
? (text) => spinner.setText(renderStatusFromText(text))
|
|
310
|
+
: null,
|
|
311
|
+
});
|
|
312
|
+
if (slidesOutput) {
|
|
313
|
+
const existingSlidesExtracted = hooks.onSlidesExtracted;
|
|
314
|
+
const existingSlidesDone = hooks.onSlidesDone;
|
|
315
|
+
const existingSlideChunk = hooks.onSlideChunk;
|
|
316
|
+
hooks.onSlidesExtracted = (value) => {
|
|
317
|
+
existingSlidesExtracted?.(value);
|
|
318
|
+
slidesOutput.onSlidesExtracted(value);
|
|
319
|
+
};
|
|
320
|
+
hooks.onSlidesDone = (result) => {
|
|
321
|
+
existingSlidesDone?.(result);
|
|
322
|
+
slidesOutput.onSlidesDone(result);
|
|
323
|
+
};
|
|
324
|
+
hooks.onSlideChunk = (chunk) => {
|
|
325
|
+
existingSlideChunk?.(chunk);
|
|
326
|
+
slidesOutput.onSlideChunk(chunk);
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
const markSlidesDone = (result) => {
|
|
330
|
+
if (slidesDone)
|
|
331
|
+
return;
|
|
332
|
+
slidesDone = true;
|
|
333
|
+
hooks.onSlidesDone?.(result);
|
|
334
|
+
};
|
|
335
|
+
const runSlidesExtraction = async () => {
|
|
336
|
+
if (!flags.slides)
|
|
337
|
+
return null;
|
|
338
|
+
if (slidesExtracted) {
|
|
339
|
+
if (!slidesDone)
|
|
340
|
+
markSlidesDone({ ok: true });
|
|
341
|
+
return slidesExtracted;
|
|
342
|
+
}
|
|
343
|
+
let errorMessage = null;
|
|
344
|
+
try {
|
|
345
|
+
const source = resolveSlideSource({ url, extracted });
|
|
346
|
+
if (!source) {
|
|
347
|
+
throw new Error("Slides are only supported for YouTube or direct video URLs.");
|
|
348
|
+
}
|
|
349
|
+
const slidesCacheKey = cacheStore && cacheState.mode === "default"
|
|
350
|
+
? buildSlidesCacheKey({ url: source.url, settings: flags.slides })
|
|
351
|
+
: null;
|
|
352
|
+
if (slidesCacheKey && cacheStore) {
|
|
353
|
+
const cached = cacheStore.getJson("slides", slidesCacheKey);
|
|
354
|
+
const validated = cached
|
|
355
|
+
? await validateSlidesCache({ cached, source, settings: flags.slides })
|
|
356
|
+
: null;
|
|
357
|
+
if (validated) {
|
|
358
|
+
writeVerbose(io.stderr, flags.verbose, "cache hit slides", flags.verboseColor, io.envForRun);
|
|
359
|
+
slidesExtracted = validated;
|
|
360
|
+
resolveTimeline(validated);
|
|
361
|
+
ctx.hooks.onSlidesExtracted?.(slidesExtracted);
|
|
362
|
+
ctx.hooks.onSlidesProgress?.("Slides: cached 100%");
|
|
363
|
+
return slidesExtracted;
|
|
364
|
+
}
|
|
365
|
+
writeVerbose(io.stderr, flags.verbose, "cache miss slides", flags.verboseColor, io.envForRun);
|
|
366
|
+
}
|
|
367
|
+
if (flags.progressEnabled) {
|
|
368
|
+
spinner.setText(renderStatus("Extracting slides"));
|
|
369
|
+
oscProgress.setIndeterminate("Extracting slides");
|
|
370
|
+
}
|
|
371
|
+
// Prefer indeterminate progress until we get real percentage updates from the slide pipeline.
|
|
372
|
+
ctx.hooks.onSlidesProgress?.("Slides: extracting");
|
|
373
|
+
const onSlidesLog = (message) => {
|
|
374
|
+
writeVerbose(io.stderr, flags.verbose, `slides ${message}`, flags.verboseColor, io.envForRun);
|
|
375
|
+
};
|
|
376
|
+
slidesExtracted = await extractSlidesForSource({
|
|
377
|
+
source,
|
|
378
|
+
settings: flags.slides,
|
|
379
|
+
noCache: cacheState.mode === "bypass",
|
|
380
|
+
mediaCache: ctx.mediaCache,
|
|
381
|
+
env: io.env,
|
|
382
|
+
timeoutMs: flags.timeoutMs,
|
|
383
|
+
ytDlpPath: model.apiStatus.ytDlpPath,
|
|
384
|
+
ytDlpCookiesFromBrowser: model.apiStatus.ytDlpCookiesFromBrowser,
|
|
385
|
+
ffmpegPath: null,
|
|
386
|
+
tesseractPath: null,
|
|
387
|
+
hooks: {
|
|
388
|
+
onSlideChunk: (chunk) => ctx.hooks.onSlideChunk?.(chunk),
|
|
389
|
+
onSlidesTimeline: (timeline) => {
|
|
390
|
+
resolveTimeline(timeline);
|
|
391
|
+
ctx.hooks.onSlidesExtracted?.(timeline);
|
|
392
|
+
},
|
|
393
|
+
onSlidesProgress: ctx.hooks.onSlidesProgress ?? undefined,
|
|
394
|
+
onSlidesLog,
|
|
395
|
+
},
|
|
396
|
+
});
|
|
397
|
+
if (slidesExtracted) {
|
|
398
|
+
ctx.hooks.onSlidesExtracted?.(slidesExtracted);
|
|
399
|
+
ctx.hooks.onSlidesProgress?.(`Slides: done (${slidesExtracted.slides.length.toString()} slides) 100%`);
|
|
400
|
+
if (slidesCacheKey && cacheStore) {
|
|
401
|
+
cacheStore.setJson("slides", slidesCacheKey, slidesExtracted, cacheState.ttlMs);
|
|
402
|
+
writeVerbose(io.stderr, flags.verbose, "cache write slides", flags.verboseColor, io.envForRun);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
if (flags.progressEnabled) {
|
|
406
|
+
updateSummaryProgress();
|
|
407
|
+
}
|
|
408
|
+
return slidesExtracted;
|
|
409
|
+
}
|
|
410
|
+
catch (error) {
|
|
411
|
+
errorMessage = error instanceof Error ? error.message : String(error);
|
|
412
|
+
throw error;
|
|
413
|
+
}
|
|
414
|
+
finally {
|
|
415
|
+
if (!slidesTimelineResolved) {
|
|
416
|
+
resolveTimeline(slidesExtracted ?? null);
|
|
417
|
+
}
|
|
418
|
+
if (!slidesDone) {
|
|
419
|
+
markSlidesDone(errorMessage ? { ok: false, error: errorMessage } : { ok: true });
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
};
|
|
423
|
+
const formatSummaryProgress = (modelId) => {
|
|
424
|
+
const dim = (value) => theme.dim(value);
|
|
425
|
+
const accent = (value) => theme.accent(value);
|
|
426
|
+
const sentLabel = `${dim("sent ")}${extractionUi.contentSizeLabel}${extractionUi.viaSourceLabel}`;
|
|
427
|
+
const modelLabel = modelId ? `${dim("model: ")}${accent(modelId)}` : "";
|
|
428
|
+
const meta = modelLabel ? `${sentLabel}${dim(", ")}${modelLabel}` : sentLabel;
|
|
429
|
+
return `${styleLabel("Summarizing")} ${dim("(")}${meta}${dim(")")}${dim("…")}`;
|
|
430
|
+
};
|
|
153
431
|
const updateSummaryProgress = () => {
|
|
154
432
|
if (!flags.progressEnabled)
|
|
155
433
|
return;
|
|
156
434
|
websiteProgress?.stop?.();
|
|
157
435
|
if (!flags.extractMode) {
|
|
158
|
-
oscProgress.setIndeterminate(
|
|
436
|
+
oscProgress.setIndeterminate("Summarizing");
|
|
159
437
|
}
|
|
160
438
|
spinner.setText(flags.extractMode
|
|
161
|
-
? `
|
|
162
|
-
:
|
|
439
|
+
? `${styleLabel("Extracted")}${styleDim(` (${extractionUi.contentSizeLabel}${extractionUi.viaSourceLabel})`)}`
|
|
440
|
+
: formatSummaryProgress());
|
|
163
441
|
};
|
|
164
442
|
updateSummaryProgress();
|
|
165
443
|
logExtractionDiagnostics({
|
|
@@ -167,41 +445,43 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
167
445
|
stderr: io.stderr,
|
|
168
446
|
verbose: flags.verbose,
|
|
169
447
|
verboseColor: flags.verboseColor,
|
|
448
|
+
env: io.envForRun,
|
|
170
449
|
});
|
|
171
450
|
const transcriptCacheStatus = extracted.diagnostics?.transcript?.cacheStatus;
|
|
172
|
-
if (transcriptCacheStatus && transcriptCacheStatus !==
|
|
173
|
-
writeVerbose(io.stderr, flags.verbose, `cache ${transcriptCacheStatus} transcript`, flags.verboseColor);
|
|
451
|
+
if (transcriptCacheStatus && transcriptCacheStatus !== "unknown") {
|
|
452
|
+
writeVerbose(io.stderr, flags.verbose, `cache ${transcriptCacheStatus} transcript`, flags.verboseColor, io.envForRun);
|
|
174
453
|
}
|
|
175
454
|
if (flags.extractMode &&
|
|
176
455
|
markdown.markdownRequested &&
|
|
177
|
-
flags.preprocessMode !==
|
|
178
|
-
markdown.effectiveMarkdownMode ===
|
|
456
|
+
flags.preprocessMode !== "off" &&
|
|
457
|
+
markdown.effectiveMarkdownMode === "auto" &&
|
|
179
458
|
!extracted.diagnostics.markdown.used &&
|
|
180
459
|
!hasUvxCli(io.env)) {
|
|
181
460
|
io.stderr.write(`${UVX_TIP}\n`);
|
|
182
461
|
}
|
|
183
462
|
if (!isYoutubeUrl && extracted.isVideoOnly && extracted.video) {
|
|
184
|
-
if (extracted.video.kind ===
|
|
185
|
-
writeVerbose(io.stderr, flags.verbose, `video-only page detected; switching to YouTube URL ${extracted.video.url}`, flags.verboseColor);
|
|
463
|
+
if (extracted.video.kind === "youtube") {
|
|
464
|
+
writeVerbose(io.stderr, flags.verbose, `video-only page detected; switching to YouTube URL ${extracted.video.url}`, flags.verboseColor, io.envForRun);
|
|
186
465
|
if (flags.progressEnabled) {
|
|
187
|
-
spinner.setText(
|
|
466
|
+
spinner.setText(renderStatus("Video-only page", ": fetching YouTube transcript…"));
|
|
188
467
|
}
|
|
189
468
|
extracted = await fetchWithCache(extracted.video.url);
|
|
190
469
|
extractionUi = deriveExtractionUi(extracted);
|
|
191
470
|
updateSummaryProgress();
|
|
192
471
|
}
|
|
193
|
-
else if (extracted.video.kind ===
|
|
194
|
-
const
|
|
472
|
+
else if (extracted.video.kind === "direct") {
|
|
473
|
+
const directVideoSlides = await runSlidesExtraction();
|
|
474
|
+
const wantsVideoUnderstanding = flags.videoMode === "understand" || flags.videoMode === "auto";
|
|
195
475
|
// Direct video URLs require a model that can consume video attachments (currently Gemini).
|
|
196
476
|
const canVideoUnderstand = wantsVideoUnderstanding &&
|
|
197
477
|
model.apiStatus.googleConfigured &&
|
|
198
|
-
(model.requestedModel.kind ===
|
|
199
|
-
(model.fixedModelSpec?.transport ===
|
|
200
|
-
model.fixedModelSpec.provider ===
|
|
478
|
+
(model.requestedModel.kind === "auto" ||
|
|
479
|
+
(model.fixedModelSpec?.transport === "native" &&
|
|
480
|
+
model.fixedModelSpec.provider === "google"));
|
|
201
481
|
if (canVideoUnderstand) {
|
|
202
482
|
hooks.onExtracted?.(extracted);
|
|
203
483
|
if (flags.progressEnabled)
|
|
204
|
-
spinner.setText(
|
|
484
|
+
spinner.setText(renderStatus("Downloading video"));
|
|
205
485
|
const loadedVideo = await loadRemoteAsset({
|
|
206
486
|
url: extracted.video.url,
|
|
207
487
|
fetchImpl: io.fetch,
|
|
@@ -210,27 +490,43 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
210
490
|
assertAssetMediaTypeSupported({ attachment: loadedVideo.attachment, sizeLabel: null });
|
|
211
491
|
let chosenModel = null;
|
|
212
492
|
if (flags.progressEnabled)
|
|
213
|
-
spinner.setText(
|
|
493
|
+
spinner.setText(renderStatus("Summarizing video"));
|
|
214
494
|
await hooks.summarizeAsset({
|
|
215
|
-
sourceKind:
|
|
495
|
+
sourceKind: "asset-url",
|
|
216
496
|
sourceLabel: loadedVideo.sourceLabel,
|
|
217
497
|
attachment: loadedVideo.attachment,
|
|
218
498
|
onModelChosen: (modelId) => {
|
|
219
499
|
chosenModel = modelId;
|
|
220
500
|
hooks.onModelChosen?.(modelId);
|
|
221
|
-
if (flags.progressEnabled)
|
|
222
|
-
|
|
501
|
+
if (flags.progressEnabled) {
|
|
502
|
+
const meta = `${styleDim("(")}${styleDim("model: ")}${theme.accent(modelId)}${styleDim(")")}`;
|
|
503
|
+
spinner.setText(renderStatusWithMeta("Summarizing video", meta));
|
|
504
|
+
}
|
|
223
505
|
},
|
|
224
506
|
});
|
|
507
|
+
const slideCount = directVideoSlides ? directVideoSlides.slides.length : null;
|
|
225
508
|
hooks.writeViaFooter([
|
|
226
509
|
...extractionUi.footerParts,
|
|
227
510
|
...(chosenModel ? [`model ${chosenModel}`] : []),
|
|
511
|
+
...(slideCount != null ? [`slides ${slideCount}`] : []),
|
|
228
512
|
]);
|
|
229
513
|
return;
|
|
230
514
|
}
|
|
231
515
|
}
|
|
232
516
|
}
|
|
517
|
+
// Start slides in parallel; wait for real timing data before prompting.
|
|
518
|
+
if (flags.slides) {
|
|
519
|
+
void runSlidesExtraction().catch((error) => {
|
|
520
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
521
|
+
ctx.hooks.onSlidesProgress?.(`Slides: failed (${message})`);
|
|
522
|
+
writeVerbose(io.stderr, flags.verbose, `slides failed: ${message}`, flags.verboseColor, io.envForRun);
|
|
523
|
+
});
|
|
524
|
+
}
|
|
233
525
|
hooks.onExtracted?.(extracted);
|
|
526
|
+
let slidesForPrompt = null;
|
|
527
|
+
if (slidesTimelinePromise) {
|
|
528
|
+
slidesForPrompt = await slidesTimelinePromise;
|
|
529
|
+
}
|
|
234
530
|
const prompt = buildUrlPrompt({
|
|
235
531
|
extracted,
|
|
236
532
|
outputLanguage: flags.outputLanguage,
|
|
@@ -238,6 +534,7 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
238
534
|
promptOverride: flags.promptOverride ?? null,
|
|
239
535
|
lengthInstruction: flags.lengthInstruction ?? null,
|
|
240
536
|
languageInstruction: flags.languageInstruction ?? null,
|
|
537
|
+
slides: slidesForPrompt ?? slidesExtracted ?? null,
|
|
241
538
|
});
|
|
242
539
|
// Whisper transcription costs need to be folded into the finish line totals.
|
|
243
540
|
const transcriptionCostUsd = estimateWhisperTranscriptionCostUsd({
|
|
@@ -246,20 +543,21 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
246
543
|
mediaDurationSeconds: extracted.mediaDurationSeconds,
|
|
247
544
|
openaiWhisperUsdPerMinute: model.openaiWhisperUsdPerMinute,
|
|
248
545
|
});
|
|
249
|
-
const transcriptionCostLabel = typeof transcriptionCostUsd ===
|
|
546
|
+
const transcriptionCostLabel = typeof transcriptionCostUsd === "number" ? `txcost=${formatUSD(transcriptionCostUsd)}` : null;
|
|
250
547
|
hooks.setTranscriptionCost(transcriptionCostUsd, transcriptionCostLabel);
|
|
251
548
|
if (flags.extractMode) {
|
|
252
549
|
// Apply transcript→markdown conversion if requested
|
|
253
550
|
let extractedForOutput = extracted;
|
|
254
551
|
if (markdown.transcriptMarkdownRequested && markdown.convertTranscriptToMarkdown) {
|
|
255
552
|
if (flags.progressEnabled) {
|
|
256
|
-
spinner.setText(
|
|
553
|
+
spinner.setText(renderStatus("Converting transcript to markdown"));
|
|
257
554
|
}
|
|
258
555
|
const markdownContent = await markdown.convertTranscriptToMarkdown({
|
|
259
556
|
title: extracted.title,
|
|
260
557
|
source: extracted.siteName,
|
|
261
558
|
transcript: extracted.content,
|
|
262
559
|
timeoutMs: flags.timeoutMs,
|
|
560
|
+
outputLanguage: flags.outputLanguage,
|
|
263
561
|
});
|
|
264
562
|
extractedForOutput = {
|
|
265
563
|
...extracted,
|
|
@@ -270,8 +568,8 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
270
568
|
...extracted.diagnostics.markdown,
|
|
271
569
|
requested: true,
|
|
272
570
|
used: true,
|
|
273
|
-
provider:
|
|
274
|
-
notes:
|
|
571
|
+
provider: "llm",
|
|
572
|
+
notes: "transcript",
|
|
275
573
|
},
|
|
276
574
|
},
|
|
277
575
|
};
|
|
@@ -285,6 +583,8 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
285
583
|
prompt,
|
|
286
584
|
effectiveMarkdownMode: markdown.effectiveMarkdownMode,
|
|
287
585
|
transcriptionCostLabel,
|
|
586
|
+
slides: slidesExtracted ?? slidesForPrompt ?? null,
|
|
587
|
+
slidesOutput,
|
|
288
588
|
});
|
|
289
589
|
return;
|
|
290
590
|
}
|
|
@@ -292,7 +592,7 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
292
592
|
hooks.onModelChosen?.(modelId);
|
|
293
593
|
if (!flags.progressEnabled)
|
|
294
594
|
return;
|
|
295
|
-
spinner.setText(
|
|
595
|
+
spinner.setText(formatSummaryProgress(modelId));
|
|
296
596
|
};
|
|
297
597
|
await summarizeExtractedUrl({
|
|
298
598
|
ctx,
|
|
@@ -303,10 +603,16 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
303
603
|
effectiveMarkdownMode: markdown.effectiveMarkdownMode,
|
|
304
604
|
transcriptionCostLabel,
|
|
305
605
|
onModelChosen,
|
|
606
|
+
slides: slidesExtracted ?? slidesForPrompt ?? null,
|
|
607
|
+
slidesOutput,
|
|
306
608
|
});
|
|
307
609
|
}
|
|
308
610
|
finally {
|
|
309
|
-
|
|
611
|
+
if (flags.progressEnabled) {
|
|
612
|
+
process.off("SIGINT", handleSigint);
|
|
613
|
+
process.off("SIGTERM", handleSigterm);
|
|
614
|
+
}
|
|
615
|
+
hooks.clearProgressIfCurrent(pauseProgressLine);
|
|
310
616
|
stopProgress();
|
|
311
617
|
}
|
|
312
618
|
}
|