@steipete/summarize 0.10.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +80 -28
- package/README.md +115 -30
- package/dist/cli.js +1 -1
- package/dist/esm/cache.js +67 -65
- package/dist/esm/cache.js.map +1 -1
- package/dist/esm/cli-main.js +27 -27
- package/dist/esm/cli-main.js.map +1 -1
- package/dist/esm/cli.js +2 -2
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/config.js +310 -166
- package/dist/esm/config.js.map +1 -1
- package/dist/esm/content/asset.js +53 -50
- package/dist/esm/content/asset.js.map +1 -1
- package/dist/esm/content/index.js +1 -1
- package/dist/esm/content/index.js.map +1 -1
- package/dist/esm/costs.js +1 -1
- package/dist/esm/costs.js.map +1 -1
- package/dist/esm/daemon/agent.js +165 -164
- package/dist/esm/daemon/agent.js.map +1 -1
- package/dist/esm/daemon/auto-mode.js +3 -3
- package/dist/esm/daemon/auto-mode.js.map +1 -1
- package/dist/esm/daemon/chat.js +16 -14
- package/dist/esm/daemon/chat.js.map +1 -1
- package/dist/esm/daemon/cli-entrypoint.js +72 -0
- package/dist/esm/daemon/cli-entrypoint.js.map +1 -0
- package/dist/esm/daemon/cli.js +63 -87
- package/dist/esm/daemon/cli.js.map +1 -1
- package/dist/esm/daemon/config.js +15 -15
- package/dist/esm/daemon/config.js.map +1 -1
- package/dist/esm/daemon/constants.js +6 -6
- package/dist/esm/daemon/constants.js.map +1 -1
- package/dist/esm/daemon/env-merge.js.map +1 -1
- package/dist/esm/daemon/env-snapshot.js +36 -31
- package/dist/esm/daemon/env-snapshot.js.map +1 -1
- package/dist/esm/daemon/flow-context.js +59 -28
- package/dist/esm/daemon/flow-context.js.map +1 -1
- package/dist/esm/daemon/launchd.js +100 -55
- package/dist/esm/daemon/launchd.js.map +1 -1
- package/dist/esm/daemon/meta.js +5 -5
- package/dist/esm/daemon/meta.js.map +1 -1
- package/dist/esm/daemon/models.js +54 -31
- package/dist/esm/daemon/models.js.map +1 -1
- package/dist/esm/daemon/process-registry.js +15 -15
- package/dist/esm/daemon/process-registry.js.map +1 -1
- package/dist/esm/daemon/schtasks.js +42 -42
- package/dist/esm/daemon/schtasks.js.map +1 -1
- package/dist/esm/daemon/server.js +248 -244
- package/dist/esm/daemon/server.js.map +1 -1
- package/dist/esm/daemon/summarize-progress.js +11 -11
- package/dist/esm/daemon/summarize-progress.js.map +1 -1
- package/dist/esm/daemon/summarize.js +29 -29
- package/dist/esm/daemon/summarize.js.map +1 -1
- package/dist/esm/daemon/systemd.js +47 -47
- package/dist/esm/daemon/systemd.js.map +1 -1
- package/dist/esm/firecrawl.js +12 -12
- package/dist/esm/firecrawl.js.map +1 -1
- package/dist/esm/flags.js +32 -32
- package/dist/esm/flags.js.map +1 -1
- package/dist/esm/index.js +3 -3
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/language.js +1 -1
- package/dist/esm/language.js.map +1 -1
- package/dist/esm/llm/cli.js +128 -64
- package/dist/esm/llm/cli.js.map +1 -1
- package/dist/esm/llm/errors.js +1 -1
- package/dist/esm/llm/errors.js.map +1 -1
- package/dist/esm/llm/generate-text.js +107 -98
- package/dist/esm/llm/generate-text.js.map +1 -1
- package/dist/esm/llm/google-models.js +17 -17
- package/dist/esm/llm/google-models.js.map +1 -1
- package/dist/esm/llm/html-to-markdown.js +3 -3
- package/dist/esm/llm/html-to-markdown.js.map +1 -1
- package/dist/esm/llm/model-id.js +38 -16
- package/dist/esm/llm/model-id.js.map +1 -1
- package/dist/esm/llm/prompt.js +5 -5
- package/dist/esm/llm/prompt.js.map +1 -1
- package/dist/esm/llm/providers/anthropic.js +33 -33
- package/dist/esm/llm/providers/anthropic.js.map +1 -1
- package/dist/esm/llm/providers/google.js +19 -19
- package/dist/esm/llm/providers/google.js.map +1 -1
- package/dist/esm/llm/providers/models.js +30 -30
- package/dist/esm/llm/providers/models.js.map +1 -1
- package/dist/esm/llm/providers/openai.js +35 -34
- package/dist/esm/llm/providers/openai.js.map +1 -1
- package/dist/esm/llm/providers/shared.js +8 -8
- package/dist/esm/llm/providers/shared.js.map +1 -1
- package/dist/esm/llm/transcript-to-markdown.js +9 -5
- package/dist/esm/llm/transcript-to-markdown.js.map +1 -1
- package/dist/esm/llm/usage.js +18 -18
- package/dist/esm/llm/usage.js.map +1 -1
- package/dist/esm/logging/daemon.js +21 -21
- package/dist/esm/logging/daemon.js.map +1 -1
- package/dist/esm/logging/ring-file.js +5 -5
- package/dist/esm/logging/ring-file.js.map +1 -1
- package/dist/esm/markitdown.js +21 -19
- package/dist/esm/markitdown.js.map +1 -1
- package/dist/esm/media-cache.js +39 -39
- package/dist/esm/media-cache.js.map +1 -1
- package/dist/esm/model-auto.js +175 -106
- package/dist/esm/model-auto.js.map +1 -1
- package/dist/esm/model-spec.js +52 -42
- package/dist/esm/model-spec.js.map +1 -1
- package/dist/esm/pricing/litellm.js +4 -4
- package/dist/esm/pricing/litellm.js.map +1 -1
- package/dist/esm/processes.js +1 -1
- package/dist/esm/processes.js.map +1 -1
- package/dist/esm/prompts/index.js +1 -1
- package/dist/esm/prompts/index.js.map +1 -1
- package/dist/esm/refresh-free.js +81 -81
- package/dist/esm/refresh-free.js.map +1 -1
- package/dist/esm/run/attachments.js +47 -44
- package/dist/esm/run/attachments.js.map +1 -1
- package/dist/esm/run/bird.js +26 -26
- package/dist/esm/run/bird.js.map +1 -1
- package/dist/esm/run/cache-state.js +7 -7
- package/dist/esm/run/cache-state.js.map +1 -1
- package/dist/esm/run/cli-fallback-state.js +45 -0
- package/dist/esm/run/cli-fallback-state.js.map +1 -0
- package/dist/esm/run/cli-preflight.js +24 -24
- package/dist/esm/run/cli-preflight.js.map +1 -1
- package/dist/esm/run/constants.js +12 -12
- package/dist/esm/run/constants.js.map +1 -1
- package/dist/esm/run/cookies/twitter.js +47 -47
- package/dist/esm/run/cookies/twitter.js.map +1 -1
- package/dist/esm/run/env.js +21 -15
- package/dist/esm/run/env.js.map +1 -1
- package/dist/esm/run/fetch-with-timeout.js +4 -4
- package/dist/esm/run/fetch-with-timeout.js.map +1 -1
- package/dist/esm/run/finish-line.js +68 -68
- package/dist/esm/run/finish-line.js.map +1 -1
- package/dist/esm/run/flows/asset/extract.js +15 -15
- package/dist/esm/run/flows/asset/extract.js.map +1 -1
- package/dist/esm/run/flows/asset/input.js +47 -66
- package/dist/esm/run/flows/asset/input.js.map +1 -1
- package/dist/esm/run/flows/asset/media-policy.js +1 -1
- package/dist/esm/run/flows/asset/media-policy.js.map +1 -1
- package/dist/esm/run/flows/asset/media.js +49 -40
- package/dist/esm/run/flows/asset/media.js.map +1 -1
- package/dist/esm/run/flows/asset/output.js +12 -12
- package/dist/esm/run/flows/asset/output.js.map +1 -1
- package/dist/esm/run/flows/asset/preprocess.js +79 -44
- package/dist/esm/run/flows/asset/preprocess.js.map +1 -1
- package/dist/esm/run/flows/asset/summary.js +173 -106
- package/dist/esm/run/flows/asset/summary.js.map +1 -1
- package/dist/esm/run/flows/url/extract.js +26 -26
- package/dist/esm/run/flows/url/extract.js.map +1 -1
- package/dist/esm/run/flows/url/flow.js +104 -98
- package/dist/esm/run/flows/url/flow.js.map +1 -1
- package/dist/esm/run/flows/url/markdown.js +57 -57
- package/dist/esm/run/flows/url/markdown.js.map +1 -1
- package/dist/esm/run/flows/url/slides-output.js +61 -59
- package/dist/esm/run/flows/url/slides-output.js.map +1 -1
- package/dist/esm/run/flows/url/slides-text.js +85 -85
- package/dist/esm/run/flows/url/slides-text.js.map +1 -1
- package/dist/esm/run/flows/url/summary.js +174 -107
- package/dist/esm/run/flows/url/summary.js.map +1 -1
- package/dist/esm/run/format.js +10 -10
- package/dist/esm/run/format.js.map +1 -1
- package/dist/esm/run/help.js +141 -135
- package/dist/esm/run/help.js.map +1 -1
- package/dist/esm/run/logging.js +10 -10
- package/dist/esm/run/logging.js.map +1 -1
- package/dist/esm/run/markdown.js +12 -12
- package/dist/esm/run/markdown.js.map +1 -1
- package/dist/esm/run/media-cache-state.js +5 -5
- package/dist/esm/run/media-cache-state.js.map +1 -1
- package/dist/esm/run/model-attempts.js.map +1 -1
- package/dist/esm/run/openrouter.js +11 -11
- package/dist/esm/run/openrouter.js.map +1 -1
- package/dist/esm/run/progress.js +1 -1
- package/dist/esm/run/progress.js.map +1 -1
- package/dist/esm/run/run-config.js +16 -16
- package/dist/esm/run/run-config.js.map +1 -1
- package/dist/esm/run/run-context.js +2 -2
- package/dist/esm/run/run-context.js.map +1 -1
- package/dist/esm/run/run-env.js +55 -54
- package/dist/esm/run/run-env.js.map +1 -1
- package/dist/esm/run/run-input.js +3 -3
- package/dist/esm/run/run-input.js.map +1 -1
- package/dist/esm/run/run-metrics.js +16 -16
- package/dist/esm/run/run-metrics.js.map +1 -1
- package/dist/esm/run/run-models.js +28 -23
- package/dist/esm/run/run-models.js.map +1 -1
- package/dist/esm/run/run-output.js +3 -3
- package/dist/esm/run/run-output.js.map +1 -1
- package/dist/esm/run/run-settings.js +83 -34
- package/dist/esm/run/run-settings.js.map +1 -1
- package/dist/esm/run/run-stream.js +4 -4
- package/dist/esm/run/run-stream.js.map +1 -1
- package/dist/esm/run/runner.js +166 -127
- package/dist/esm/run/runner.js.map +1 -1
- package/dist/esm/run/slides-cli.js +43 -42
- package/dist/esm/run/slides-cli.js.map +1 -1
- package/dist/esm/run/slides-render.js +36 -36
- package/dist/esm/run/slides-render.js.map +1 -1
- package/dist/esm/run/stdin-temp-file.js +77 -0
- package/dist/esm/run/stdin-temp-file.js.map +1 -0
- package/dist/esm/run/stream-output.js +7 -7
- package/dist/esm/run/stream-output.js.map +1 -1
- package/dist/esm/run/streaming.js +16 -16
- package/dist/esm/run/streaming.js.map +1 -1
- package/dist/esm/run/summary-engine.js +57 -51
- package/dist/esm/run/summary-engine.js.map +1 -1
- package/dist/esm/run/summary-llm.js +3 -3
- package/dist/esm/run/summary-llm.js.map +1 -1
- package/dist/esm/run/terminal.js +4 -4
- package/dist/esm/run/terminal.js.map +1 -1
- package/dist/esm/run/tips.js +2 -2
- package/dist/esm/run/tips.js.map +1 -1
- package/dist/esm/run/transcriber-cli.js +49 -49
- package/dist/esm/run/transcriber-cli.js.map +1 -1
- package/dist/esm/run.js +1 -1
- package/dist/esm/run.js.map +1 -1
- package/dist/esm/shared/contracts.js +1 -1
- package/dist/esm/shared/contracts.js.map +1 -1
- package/dist/esm/shared/sse-events.js +16 -16
- package/dist/esm/shared/sse-events.js.map +1 -1
- package/dist/esm/shared/streaming-merge.js +3 -3
- package/dist/esm/shared/streaming-merge.js.map +1 -1
- package/dist/esm/slides/extract.js +258 -249
- package/dist/esm/slides/extract.js.map +1 -1
- package/dist/esm/slides/index.js +3 -3
- package/dist/esm/slides/index.js.map +1 -1
- package/dist/esm/slides/settings.js +14 -14
- package/dist/esm/slides/settings.js.map +1 -1
- package/dist/esm/slides/store.js +9 -9
- package/dist/esm/slides/store.js.map +1 -1
- package/dist/esm/tty/format.js +13 -13
- package/dist/esm/tty/format.js.map +1 -1
- package/dist/esm/tty/osc-progress.js +1 -1
- package/dist/esm/tty/osc-progress.js.map +1 -1
- package/dist/esm/tty/progress/fetch-html.js +14 -14
- package/dist/esm/tty/progress/fetch-html.js.map +1 -1
- package/dist/esm/tty/progress/transcript.js +70 -62
- package/dist/esm/tty/progress/transcript.js.map +1 -1
- package/dist/esm/tty/spinner.js +20 -9
- package/dist/esm/tty/spinner.js.map +1 -1
- package/dist/esm/tty/theme.js +92 -92
- package/dist/esm/tty/theme.js.map +1 -1
- package/dist/esm/tty/website-progress.js +32 -32
- package/dist/esm/tty/website-progress.js.map +1 -1
- package/dist/esm/version.js +29 -29
- package/dist/esm/version.js.map +1 -1
- package/dist/types/cache.d.ts +6 -6
- package/dist/types/config.d.ts +49 -7
- package/dist/types/content/asset.d.ts +8 -6
- package/dist/types/content/index.d.ts +1 -1
- package/dist/types/costs.d.ts +3 -3
- package/dist/types/daemon/agent.d.ts +1 -1
- package/dist/types/daemon/auto-mode.d.ts +3 -3
- package/dist/types/daemon/chat.d.ts +2 -2
- package/dist/types/daemon/cli-entrypoint.d.ts +2 -0
- package/dist/types/daemon/config.d.ts +2 -2
- package/dist/types/daemon/env-merge.d.ts +1 -1
- package/dist/types/daemon/env-snapshot.d.ts +1 -1
- package/dist/types/daemon/flow-context.d.ts +7 -7
- package/dist/types/daemon/launchd.d.ts +8 -0
- package/dist/types/daemon/models.d.ts +6 -2
- package/dist/types/daemon/process-registry.d.ts +5 -5
- package/dist/types/daemon/server.d.ts +2 -2
- package/dist/types/daemon/summarize-progress.d.ts +1 -1
- package/dist/types/daemon/summarize.d.ts +7 -7
- package/dist/types/firecrawl.d.ts +1 -1
- package/dist/types/flags.d.ts +11 -11
- package/dist/types/index.d.ts +4 -4
- package/dist/types/language.d.ts +1 -1
- package/dist/types/llm/attachments.d.ts +1 -1
- package/dist/types/llm/cli.d.ts +3 -3
- package/dist/types/llm/generate-text.d.ts +7 -7
- package/dist/types/llm/html-to-markdown.d.ts +3 -3
- package/dist/types/llm/model-id.d.ts +1 -1
- package/dist/types/llm/prompt.d.ts +2 -2
- package/dist/types/llm/providers/anthropic.d.ts +3 -3
- package/dist/types/llm/providers/google.d.ts +3 -3
- package/dist/types/llm/providers/models.d.ts +2 -2
- package/dist/types/llm/providers/openai.d.ts +4 -4
- package/dist/types/llm/providers/shared.d.ts +2 -2
- package/dist/types/llm/transcript-to-markdown.d.ts +4 -2
- package/dist/types/llm/usage.d.ts +1 -1
- package/dist/types/logging/daemon.d.ts +4 -4
- package/dist/types/markitdown.d.ts +1 -1
- package/dist/types/media-cache.d.ts +2 -2
- package/dist/types/model-auto.d.ts +14 -4
- package/dist/types/model-spec.d.ts +10 -10
- package/dist/types/pricing/litellm.d.ts +1 -1
- package/dist/types/processes.d.ts +1 -1
- package/dist/types/prompts/index.d.ts +1 -1
- package/dist/types/run/attachments.d.ts +7 -7
- package/dist/types/run/bird.d.ts +2 -2
- package/dist/types/run/cache-state.d.ts +2 -2
- package/dist/types/run/cli-fallback-state.d.ts +6 -0
- package/dist/types/run/constants.d.ts +1 -1
- package/dist/types/run/cookies/twitter.d.ts +1 -1
- package/dist/types/run/env.d.ts +1 -1
- package/dist/types/run/finish-line.d.ts +5 -5
- package/dist/types/run/flows/asset/extract.d.ts +4 -4
- package/dist/types/run/flows/asset/input.d.ts +9 -3
- package/dist/types/run/flows/asset/media.d.ts +1 -1
- package/dist/types/run/flows/asset/output.d.ts +5 -5
- package/dist/types/run/flows/asset/preprocess.d.ts +23 -17
- package/dist/types/run/flows/asset/summary.d.ts +19 -17
- package/dist/types/run/flows/url/extract.d.ts +1 -1
- package/dist/types/run/flows/url/flow.d.ts +1 -1
- package/dist/types/run/flows/url/markdown.d.ts +6 -6
- package/dist/types/run/flows/url/slides-output.d.ts +7 -7
- package/dist/types/run/flows/url/slides-text.d.ts +9 -9
- package/dist/types/run/flows/url/summary.d.ts +11 -11
- package/dist/types/run/flows/url/types.d.ts +26 -22
- package/dist/types/run/format.d.ts +3 -3
- package/dist/types/run/help.d.ts +1 -1
- package/dist/types/run/media-cache-state.d.ts +2 -2
- package/dist/types/run/model-attempts.d.ts +1 -1
- package/dist/types/run/run-config.d.ts +4 -4
- package/dist/types/run/run-context.d.ts +3 -1
- package/dist/types/run/run-env.d.ts +3 -1
- package/dist/types/run/run-input.d.ts +2 -2
- package/dist/types/run/run-metrics.d.ts +3 -3
- package/dist/types/run/run-models.d.ts +3 -2
- package/dist/types/run/run-output.d.ts +1 -1
- package/dist/types/run/run-settings.d.ts +15 -6
- package/dist/types/run/run-stream.d.ts +2 -2
- package/dist/types/run/runner.d.ts +3 -2
- package/dist/types/run/slides-render.d.ts +4 -4
- package/dist/types/run/stdin-temp-file.d.ts +9 -0
- package/dist/types/run/stream-output.d.ts +1 -1
- package/dist/types/run/streaming.d.ts +4 -4
- package/dist/types/run/summary-engine.d.ts +11 -11
- package/dist/types/run/summary-llm.d.ts +5 -5
- package/dist/types/run/types.d.ts +4 -4
- package/dist/types/run.d.ts +1 -1
- package/dist/types/shared/contracts.d.ts +2 -2
- package/dist/types/shared/sse-events.d.ts +9 -9
- package/dist/types/slides/extract.d.ts +5 -4
- package/dist/types/slides/index.d.ts +5 -5
- package/dist/types/slides/store.d.ts +2 -2
- package/dist/types/slides/types.d.ts +2 -2
- package/dist/types/tty/osc-progress.d.ts +5 -5
- package/dist/types/tty/progress/fetch-html.d.ts +3 -3
- package/dist/types/tty/progress/transcript.d.ts +3 -3
- package/dist/types/tty/spinner.d.ts +2 -2
- package/dist/types/tty/theme.d.ts +2 -2
- package/dist/types/tty/website-progress.d.ts +3 -3
- package/dist/types/version.d.ts +1 -1
- package/docs/agent.md +38 -4
- package/docs/assets/site.js +46 -46
- package/docs/chrome-extension.md +11 -5
- package/docs/cli.md +59 -13
- package/docs/config.md +59 -10
- package/docs/extract-only.md +2 -0
- package/docs/index.html +33 -14
- package/docs/llm.md +7 -4
- package/docs/media.md +5 -4
- package/docs/model-auto.md +3 -2
- package/docs/nvidia-onnx-transcription.md +3 -3
- package/docs/openai.md +1 -1
- package/docs/releasing.md +3 -0
- package/docs/site/404.html +4 -1
- package/docs/site/assets/site.js +46 -46
- package/docs/site/docs/chrome-extension.html +18 -6
- package/docs/site/docs/config.html +29 -8
- package/docs/site/docs/extract-only.html +16 -4
- package/docs/site/docs/firecrawl.html +12 -3
- package/docs/site/docs/index.html +35 -6
- package/docs/site/docs/llm.html +19 -5
- package/docs/site/docs/openai.html +18 -5
- package/docs/site/docs/website.html +29 -9
- package/docs/site/docs/youtube.html +12 -3
- package/docs/site/index.html +33 -14
- package/docs/slides.md +13 -5
- package/docs/smoketest.md +29 -20
- package/docs/timestamps.md +21 -0
- package/docs/website.md +2 -1
- package/docs/youtube.md +4 -0
- package/package.json +36 -35
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { formatBytes } from
|
|
2
|
-
import { withBirdTip } from
|
|
3
|
-
import { buildSummaryFinishLabel } from
|
|
4
|
-
import { formatOptionalNumber, formatOptionalString } from
|
|
5
|
-
import { writeVerbose } from
|
|
1
|
+
import { formatBytes } from "../../../tty/format.js";
|
|
2
|
+
import { withBirdTip } from "../../bird.js";
|
|
3
|
+
import { buildSummaryFinishLabel } from "../../finish-line.js";
|
|
4
|
+
import { formatOptionalNumber, formatOptionalString } from "../../format.js";
|
|
5
|
+
import { writeVerbose } from "../../logging.js";
|
|
6
6
|
export async function fetchLinkContentWithBirdTip({ client, url, options, env, }) {
|
|
7
7
|
try {
|
|
8
8
|
return await client.fetchLinkContent(url, options);
|
|
@@ -12,41 +12,41 @@ export async function fetchLinkContentWithBirdTip({ client, url, options, env, }
|
|
|
12
12
|
}
|
|
13
13
|
}
|
|
14
14
|
export function deriveExtractionUi(extracted) {
|
|
15
|
-
const extractedContentBytes = Buffer.byteLength(extracted.content,
|
|
15
|
+
const extractedContentBytes = Buffer.byteLength(extracted.content, "utf8");
|
|
16
16
|
const contentSizeLabel = formatBytes(extractedContentBytes);
|
|
17
17
|
const viaSources = [];
|
|
18
|
-
if (extracted.diagnostics.strategy ===
|
|
19
|
-
viaSources.push(
|
|
18
|
+
if (extracted.diagnostics.strategy === "bird") {
|
|
19
|
+
viaSources.push("bird");
|
|
20
20
|
}
|
|
21
|
-
if (extracted.diagnostics.strategy ===
|
|
22
|
-
viaSources.push(
|
|
21
|
+
if (extracted.diagnostics.strategy === "nitter") {
|
|
22
|
+
viaSources.push("Nitter");
|
|
23
23
|
}
|
|
24
24
|
if (extracted.diagnostics.firecrawl.used) {
|
|
25
|
-
viaSources.push(
|
|
25
|
+
viaSources.push("Firecrawl");
|
|
26
26
|
}
|
|
27
|
-
const viaSourceLabel = viaSources.length > 0 ? `, ${viaSources.join(
|
|
27
|
+
const viaSourceLabel = viaSources.length > 0 ? `, ${viaSources.join("+")}` : "";
|
|
28
28
|
const footerParts = [];
|
|
29
|
-
if (extracted.diagnostics.strategy ===
|
|
30
|
-
footerParts.push(
|
|
31
|
-
if (extracted.diagnostics.strategy ===
|
|
32
|
-
footerParts.push(
|
|
33
|
-
if (extracted.diagnostics.strategy ===
|
|
34
|
-
footerParts.push(
|
|
29
|
+
if (extracted.diagnostics.strategy === "html")
|
|
30
|
+
footerParts.push("html");
|
|
31
|
+
if (extracted.diagnostics.strategy === "bird")
|
|
32
|
+
footerParts.push("bird");
|
|
33
|
+
if (extracted.diagnostics.strategy === "nitter")
|
|
34
|
+
footerParts.push("nitter");
|
|
35
35
|
if (extracted.diagnostics.firecrawl.used)
|
|
36
|
-
footerParts.push(
|
|
36
|
+
footerParts.push("firecrawl");
|
|
37
37
|
if (extracted.diagnostics.markdown.used) {
|
|
38
|
-
if (extracted.diagnostics.markdown.provider ===
|
|
39
|
-
footerParts.push(extracted.diagnostics.markdown.notes ===
|
|
38
|
+
if (extracted.diagnostics.markdown.provider === "llm") {
|
|
39
|
+
footerParts.push(extracted.diagnostics.markdown.notes === "transcript" ? "transcript→md llm" : "html→md llm");
|
|
40
40
|
}
|
|
41
41
|
else {
|
|
42
|
-
footerParts.push(
|
|
42
|
+
footerParts.push("markdown");
|
|
43
43
|
}
|
|
44
44
|
}
|
|
45
45
|
if (extracted.diagnostics.transcript.textProvided) {
|
|
46
|
-
footerParts.push(`transcript ${extracted.diagnostics.transcript.provider ??
|
|
46
|
+
footerParts.push(`transcript ${extracted.diagnostics.transcript.provider ?? "unknown"}`);
|
|
47
47
|
}
|
|
48
48
|
if (extracted.isVideoOnly && extracted.video) {
|
|
49
|
-
footerParts.push(extracted.video.kind ===
|
|
49
|
+
footerParts.push(extracted.video.kind === "youtube" ? "video youtube" : "video url");
|
|
50
50
|
}
|
|
51
51
|
const finishSourceLabel = buildSummaryFinishLabel({
|
|
52
52
|
extracted: { diagnostics: extracted.diagnostics, wordCount: extracted.wordCount },
|
|
@@ -64,7 +64,7 @@ export function logExtractionDiagnostics({ extracted, stderr, verbose, verboseCo
|
|
|
64
64
|
writeVerbose(stderr, verbose, `extract firecrawl attempted=${extracted.diagnostics.firecrawl.attempted} used=${extracted.diagnostics.firecrawl.used} notes=${formatOptionalString(extracted.diagnostics.firecrawl.notes ?? null)}`, verboseColor, env);
|
|
65
65
|
writeVerbose(stderr, verbose, `extract markdown requested=${extracted.diagnostics.markdown.requested} used=${extracted.diagnostics.markdown.used} provider=${formatOptionalString(extracted.diagnostics.markdown.provider ?? null)} notes=${formatOptionalString(extracted.diagnostics.markdown.notes ?? null)}`, verboseColor, env);
|
|
66
66
|
writeVerbose(stderr, verbose, `extract transcript textProvided=${extracted.diagnostics.transcript.textProvided} provider=${formatOptionalString(extracted.diagnostics.transcript.provider ?? null)} attemptedProviders=${extracted.diagnostics.transcript.attemptedProviders.length > 0
|
|
67
|
-
? extracted.diagnostics.transcript.attemptedProviders.join(
|
|
68
|
-
:
|
|
67
|
+
? extracted.diagnostics.transcript.attemptedProviders.join(",")
|
|
68
|
+
: "none"} notes=${formatOptionalString(extracted.diagnostics.transcript.notes ?? null)}`, verboseColor, env);
|
|
69
69
|
}
|
|
70
70
|
//# sourceMappingURL=extract.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../../src/run/flows/url/extract.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,
|
|
1
|
+
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../../src/run/flows/url/extract.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAShD,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,EAChD,MAAM,EACN,GAAG,EACH,OAAO,EACP,GAAG,GAWJ;IACC,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IACrC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,SAA+B;IAChE,MAAM,qBAAqB,GAAG,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC3E,MAAM,gBAAgB,GAAG,WAAW,CAAC,qBAAqB,CAAC,CAAC;IAE5D,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,IAAI,SAAS,CAAC,WAAW,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QAC9C,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IACD,IAAI,SAAS,CAAC,WAAW,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAChD,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IACD,IAAI,SAAS,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QACzC,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAEhF,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,IAAI,SAAS,CAAC,WAAW,CAAC,QAAQ,KAAK,MAAM;QAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxE,IAAI,SAAS,CAAC,WAAW,CAAC,QAAQ,KAAK,MAAM;QAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxE,IAAI,SAAS,CAAC,WAAW,CAAC,QAAQ,KAAK,QAAQ;QAAE,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5E,IAAI,SAAS,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI;QAAE,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACxE,IAAI,SAAS,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACxC,IAAI,SAAS,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;YACtD,WAAW,CAAC,IAAI,CACd,SAAS,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,KAAK,YAAY,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,aAAa,CAC5F,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,IAAI,SAAS,CAAC,WAAW,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC;QAClD,WAAW,CAAC,IAAI,CAAC,cAAc,SAAS,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,IAAI,SAAS,EAAE,CAAC,CAAC;IAC3F,CAAC;IACD,IAAI,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QAC7C,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IACvF,CAAC;IAED,MAAM,iBAAiB,GAAG,uBAAuB,CAAC;QAChD,SAAS,EAAE,EAAE,WAAW,EAAE,SAAS,CAAC,WAAW,EAAE,SAAS,EAAE,SAAS,CAAC,SAAS,EAAE;KAClF,CAAC,CAAC;IAEH,OAAO;QACL,gBAAgB;QAChB,cAAc;QACd,WAAW;QACX,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,EACvC,SAAS,EACT,MAAM,EACN,OAAO,EACP,YAAY,EACZ,GAAG,GAOJ;IACC,YAAY,CACV,MAAM,EACN,OAAO,EACP,yBAAyB,SAAS,CAAC,WAAW,CAAC,QAAQ,aAAa,oBAAoB,CACtF,SAAS,CAAC,QAAQ,CACnB,UAAU,oBAAoB,CAAC,SAAS,CAAC,KAAK,CAAC,qBAAqB,oBAAoB,CACvF,SAAS,CAAC,gBAAgB,CAC3B,EAAE,EACH,YAAY,EACZ,GAAG,CACJ,CAAC;IACF,YAAY,CACV,MAAM,EACN,OAAO,EACP,4BAA4B,SAAS,CAAC,eAAe,UAAU,SAAS,CAAC,SAAS,yBAAyB,oBAAoB,CAC7H,SAAS,CAAC,oBAAoB,CAC/B,oBAAoB,oBAAoB,CAAC,SAAS,CAAC,eAAe,CAAC,EAAE,EACtE,YAAY,EACZ,GAAG,CACJ,CAAC;IACF,YAAY,CACV,MAAM,EACN,OAAO,EACP,+BAA+B,SAAS,CAAC,WAAW,CAAC,SAAS,CAAC,SAAS,SAAS,SAAS,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,UAAU,oBAAoB,CACjJ,SAAS,CAAC,WAAW,CAAC,SAAS,CAAC,KAAK,IAAI,IAAI,CAC9C,EAAE,EACH,YAAY,EACZ,GAAG,CACJ,CAAC;IACF,YAAY,CACV,MAAM,EACN,OAAO,EACP,8BAA8B,SAAS,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,SAAS,SAAS,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,aAAa,oBAAoB,CACjJ,SAAS,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI,CAChD,UAAU,oBAAoB,CAAC,SAAS,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,EAC/E,YAAY,EACZ,GAAG,CACJ,CAAC;IACF,YAAY,CACV,MAAM,EACN,OAAO,EACP,mCAAmC,SAAS,CAAC,WAAW,CAAC,UAAU,CAAC,YAAY,aAAa,oBAAoB,CAC/G,SAAS,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,IAAI,IAAI,CAClD,uBACC,SAAS,CAAC,WAAW,CAAC,UAAU,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC;QAC5D,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,UAAU,CAAC,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC;QAC/D,CAAC,CAAC,MACN,UAAU,oBAAoB,CAAC,SAAS,CAAC,WAAW,CAAC,UAAU,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,EAChF,YAAY,EACZ,GAAG,CACJ,CAAC;AACJ,CAAC"}
|
|
@@ -1,27 +1,27 @@
|
|
|
1
|
-
import * as urlUtils from
|
|
2
|
-
import { buildExtractCacheKey, buildSlidesCacheKey } from
|
|
3
|
-
import { loadRemoteAsset } from
|
|
4
|
-
import { createLinkPreviewClient, } from
|
|
5
|
-
import { createFirecrawlScraper } from
|
|
6
|
-
import { extractSlidesForSource, resolveSlideSource, validateSlidesCache, } from
|
|
7
|
-
import { createOscProgressController } from
|
|
8
|
-
import { startSpinner } from
|
|
9
|
-
import { createThemeRenderer, resolveThemeNameFromSources, resolveTrueColor, } from
|
|
10
|
-
import { createWebsiteProgress } from
|
|
11
|
-
import { assertAssetMediaTypeSupported } from
|
|
12
|
-
import { readTweetWithBird } from
|
|
13
|
-
import { UVX_TIP } from
|
|
14
|
-
import { resolveTwitterCookies } from
|
|
15
|
-
import { hasBirdCli, hasUvxCli } from
|
|
16
|
-
import { estimateWhisperTranscriptionCostUsd, formatOptionalNumber, formatOptionalString, formatUSD, } from
|
|
17
|
-
import { writeVerbose } from
|
|
18
|
-
import { deriveExtractionUi, fetchLinkContentWithBirdTip, logExtractionDiagnostics, } from
|
|
19
|
-
import { createMarkdownConverters } from
|
|
20
|
-
import { createSlidesTerminalOutput } from
|
|
21
|
-
import { buildUrlPrompt, outputExtractedUrl, summarizeExtractedUrl } from
|
|
1
|
+
import * as urlUtils from "@steipete/summarize-core/content/url";
|
|
2
|
+
import { buildExtractCacheKey, buildSlidesCacheKey } from "../../../cache.js";
|
|
3
|
+
import { loadRemoteAsset } from "../../../content/asset.js";
|
|
4
|
+
import { createLinkPreviewClient, } from "../../../content/index.js";
|
|
5
|
+
import { createFirecrawlScraper } from "../../../firecrawl.js";
|
|
6
|
+
import { extractSlidesForSource, resolveSlideSource, validateSlidesCache, } from "../../../slides/index.js";
|
|
7
|
+
import { createOscProgressController } from "../../../tty/osc-progress.js";
|
|
8
|
+
import { startSpinner } from "../../../tty/spinner.js";
|
|
9
|
+
import { createThemeRenderer, resolveThemeNameFromSources, resolveTrueColor, } from "../../../tty/theme.js";
|
|
10
|
+
import { createWebsiteProgress } from "../../../tty/website-progress.js";
|
|
11
|
+
import { assertAssetMediaTypeSupported } from "../../attachments.js";
|
|
12
|
+
import { readTweetWithBird } from "../../bird.js";
|
|
13
|
+
import { UVX_TIP } from "../../constants.js";
|
|
14
|
+
import { resolveTwitterCookies } from "../../cookies/twitter.js";
|
|
15
|
+
import { hasBirdCli, hasUvxCli } from "../../env.js";
|
|
16
|
+
import { estimateWhisperTranscriptionCostUsd, formatOptionalNumber, formatOptionalString, formatUSD, } from "../../format.js";
|
|
17
|
+
import { writeVerbose } from "../../logging.js";
|
|
18
|
+
import { deriveExtractionUi, fetchLinkContentWithBirdTip, logExtractionDiagnostics, } from "./extract.js";
|
|
19
|
+
import { createMarkdownConverters } from "./markdown.js";
|
|
20
|
+
import { createSlidesTerminalOutput } from "./slides-output.js";
|
|
21
|
+
import { buildUrlPrompt, outputExtractedUrl, summarizeExtractedUrl } from "./summary.js";
|
|
22
22
|
export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
23
23
|
if (!url) {
|
|
24
|
-
throw new Error(
|
|
24
|
+
throw new Error("Only HTTP and HTTPS URLs can be summarized");
|
|
25
25
|
}
|
|
26
26
|
const { io, flags, model, cache: cacheState, hooks } = ctx;
|
|
27
27
|
const theme = createThemeRenderer({
|
|
@@ -30,17 +30,17 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
30
30
|
trueColor: resolveTrueColor(io.envForRun),
|
|
31
31
|
});
|
|
32
32
|
const markdown = createMarkdownConverters(ctx, { isYoutubeUrl });
|
|
33
|
-
if (flags.firecrawlMode ===
|
|
34
|
-
throw new Error(
|
|
33
|
+
if (flags.firecrawlMode === "always" && !model.apiStatus.firecrawlConfigured) {
|
|
34
|
+
throw new Error("--firecrawl always requires FIRECRAWL_API_KEY");
|
|
35
35
|
}
|
|
36
|
-
writeVerbose(io.stderr, flags.verbose, `config url=${url} timeoutMs=${flags.timeoutMs} youtube=${flags.youtubeMode} firecrawl=${flags.firecrawlMode} length=${flags.lengthArg.kind ===
|
|
36
|
+
writeVerbose(io.stderr, flags.verbose, `config url=${url} timeoutMs=${flags.timeoutMs} youtube=${flags.youtubeMode} firecrawl=${flags.firecrawlMode} length=${flags.lengthArg.kind === "preset"
|
|
37
37
|
? flags.lengthArg.preset
|
|
38
|
-
: `${flags.lengthArg.maxCharacters} chars`} maxOutputTokens=${formatOptionalNumber(flags.maxOutputTokensArg)} retries=${flags.retries} json=${flags.json} extract=${flags.extractMode} format=${flags.format} preprocess=${flags.preprocessMode} markdownMode=${flags.markdownMode} model=${model.requestedModelLabel} videoMode=${flags.videoMode} timestamps=${flags.transcriptTimestamps ?
|
|
38
|
+
: `${flags.lengthArg.maxCharacters} chars`} maxOutputTokens=${formatOptionalNumber(flags.maxOutputTokensArg)} retries=${flags.retries} json=${flags.json} extract=${flags.extractMode} format=${flags.format} preprocess=${flags.preprocessMode} markdownMode=${flags.markdownMode} model=${model.requestedModelLabel} videoMode=${flags.videoMode} timestamps=${flags.transcriptTimestamps ? "on" : "off"} stream=${flags.streamingEnabled ? "on" : "off"} plain=${flags.plain}`, flags.verboseColor, io.envForRun);
|
|
39
39
|
writeVerbose(io.stderr, flags.verbose, `configFile path=${formatOptionalString(flags.configPath)} model=${formatOptionalString(flags.configModelLabel)}`, flags.verboseColor, io.envForRun);
|
|
40
40
|
writeVerbose(io.stderr, flags.verbose, `env xaiKey=${Boolean(model.apiStatus.xaiApiKey)} openaiKey=${Boolean(model.apiStatus.apiKey)} zaiKey=${Boolean(model.apiStatus.zaiApiKey)} googleKey=${model.apiStatus.googleConfigured} anthropicKey=${model.apiStatus.anthropicConfigured} openrouterKey=${model.apiStatus.openrouterConfigured} apifyToken=${Boolean(model.apiStatus.apifyToken)} firecrawlKey=${model.apiStatus.firecrawlConfigured}`, flags.verboseColor, io.envForRun);
|
|
41
41
|
writeVerbose(io.stderr, flags.verbose, `markdown htmlRequested=${markdown.markdownRequested} transcriptRequested=${markdown.transcriptMarkdownRequested} provider=${markdown.markdownProvider}`, flags.verboseColor, io.envForRun);
|
|
42
42
|
const firecrawlApiKey = model.apiStatus.firecrawlApiKey;
|
|
43
|
-
const scrapeWithFirecrawl = model.apiStatus.firecrawlConfigured && flags.firecrawlMode !==
|
|
43
|
+
const scrapeWithFirecrawl = model.apiStatus.firecrawlConfigured && flags.firecrawlMode !== "off" && firecrawlApiKey
|
|
44
44
|
? createFirecrawlScraper({
|
|
45
45
|
apiKey: firecrawlApiKey,
|
|
46
46
|
fetchImpl: io.fetch,
|
|
@@ -49,24 +49,24 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
49
49
|
const readTweetWithBirdClient = hasBirdCli(io.env)
|
|
50
50
|
? ({ url, timeoutMs }) => readTweetWithBird({ url, timeoutMs, env: io.env })
|
|
51
51
|
: null;
|
|
52
|
-
writeVerbose(io.stderr, flags.verbose,
|
|
52
|
+
writeVerbose(io.stderr, flags.verbose, "extract start", flags.verboseColor, io.envForRun);
|
|
53
53
|
const oscProgress = createOscProgressController({
|
|
54
|
-
label:
|
|
54
|
+
label: "Fetching website",
|
|
55
55
|
env: io.env,
|
|
56
56
|
isTty: flags.progressEnabled,
|
|
57
57
|
write: (data) => io.stderr.write(data),
|
|
58
58
|
});
|
|
59
|
-
oscProgress.setIndeterminate(
|
|
59
|
+
oscProgress.setIndeterminate("Fetching website");
|
|
60
60
|
const spinner = startSpinner({
|
|
61
|
-
text: `${theme.label(
|
|
61
|
+
text: `${theme.label("Fetching website")}${theme.dim(" (connecting)…")}`,
|
|
62
62
|
enabled: flags.progressEnabled,
|
|
63
63
|
stream: io.stderr,
|
|
64
64
|
color: theme.palette.spinner,
|
|
65
65
|
});
|
|
66
66
|
const styleLabel = (text) => theme.label(text);
|
|
67
67
|
const styleDim = (text) => theme.dim(text);
|
|
68
|
-
const renderStatus = (label, detail =
|
|
69
|
-
const renderStatusWithMeta = (label, meta, suffix =
|
|
68
|
+
const renderStatus = (label, detail = "…") => `${styleLabel(label)}${styleDim(detail)}`;
|
|
69
|
+
const renderStatusWithMeta = (label, meta, suffix = "…") => `${styleLabel(label)} ${meta}${styleDim(suffix)}`;
|
|
70
70
|
const renderStatusFromText = (text) => {
|
|
71
71
|
const match = text.match(/^([^:]+):(.*)$/);
|
|
72
72
|
if (!match)
|
|
@@ -91,8 +91,8 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
91
91
|
process.exit(143);
|
|
92
92
|
};
|
|
93
93
|
if (flags.progressEnabled) {
|
|
94
|
-
process.once(
|
|
95
|
-
process.once(
|
|
94
|
+
process.once("SIGINT", handleSigint);
|
|
95
|
+
process.once("SIGTERM", handleSigterm);
|
|
96
96
|
}
|
|
97
97
|
if (!hooks.onSlidesProgress && flags.progressEnabled) {
|
|
98
98
|
hooks.onSlidesProgress = (text) => {
|
|
@@ -100,10 +100,10 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
100
100
|
const percent = match ? Number(match[1]) : null;
|
|
101
101
|
spinner.setText(renderStatusFromText(text));
|
|
102
102
|
if (Number.isFinite(percent) && percent !== null) {
|
|
103
|
-
oscProgress.setPercent(
|
|
103
|
+
oscProgress.setPercent("Slides", Math.max(0, Math.min(100, percent)));
|
|
104
104
|
}
|
|
105
105
|
else {
|
|
106
|
-
oscProgress.setIndeterminate(
|
|
106
|
+
oscProgress.setIndeterminate("Slides");
|
|
107
107
|
}
|
|
108
108
|
};
|
|
109
109
|
}
|
|
@@ -113,14 +113,18 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
113
113
|
oscProgress,
|
|
114
114
|
theme,
|
|
115
115
|
});
|
|
116
|
-
const cacheStore = cacheState.mode ===
|
|
116
|
+
const cacheStore = cacheState.mode === "default" ? cacheState.store : null;
|
|
117
117
|
const transcriptCache = cacheStore ? cacheStore.transcriptCache : null;
|
|
118
118
|
const client = createLinkPreviewClient({
|
|
119
119
|
env: io.envForRun,
|
|
120
120
|
apifyApiToken: model.apiStatus.apifyToken,
|
|
121
121
|
ytDlpPath: model.apiStatus.ytDlpPath,
|
|
122
|
-
|
|
123
|
-
|
|
122
|
+
transcription: {
|
|
123
|
+
env: io.envForRun,
|
|
124
|
+
falApiKey: model.apiStatus.falApiKey,
|
|
125
|
+
groqApiKey: model.apiStatus.groqApiKey,
|
|
126
|
+
openaiApiKey: model.apiStatus.openaiTranscriptionKey,
|
|
127
|
+
},
|
|
124
128
|
scrapeWithFirecrawl,
|
|
125
129
|
convertHtmlToMarkdown: markdown.convertHtmlToMarkdown,
|
|
126
130
|
readTweetWithBird: readTweetWithBirdClient,
|
|
@@ -159,20 +163,20 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
159
163
|
try {
|
|
160
164
|
const buildFetchOptions = () => ({
|
|
161
165
|
timeoutMs: flags.timeoutMs,
|
|
162
|
-
maxCharacters: typeof flags.maxExtractCharacters ===
|
|
166
|
+
maxCharacters: typeof flags.maxExtractCharacters === "number" && flags.maxExtractCharacters > 0
|
|
163
167
|
? flags.maxExtractCharacters
|
|
164
168
|
: undefined,
|
|
165
169
|
youtubeTranscript: flags.youtubeMode,
|
|
166
|
-
mediaTranscript: flags.videoMode ===
|
|
170
|
+
mediaTranscript: flags.videoMode === "transcript" ? "prefer" : "auto",
|
|
167
171
|
transcriptTimestamps: flags.transcriptTimestamps,
|
|
168
172
|
firecrawl: flags.firecrawlMode,
|
|
169
|
-
format: markdown.markdownRequested ?
|
|
173
|
+
format: markdown.markdownRequested ? "markdown" : "text",
|
|
170
174
|
markdownMode: markdown.markdownRequested ? markdown.effectiveMarkdownMode : undefined,
|
|
171
175
|
cacheMode: cacheState.mode,
|
|
172
176
|
});
|
|
173
177
|
const fetchWithCache = async (targetUrl, { bypassExtractCache = false, } = {}) => {
|
|
174
178
|
const options = buildFetchOptions();
|
|
175
|
-
const cacheKey = cacheStore && cacheState.mode ===
|
|
179
|
+
const cacheKey = cacheStore && cacheState.mode === "default"
|
|
176
180
|
? buildExtractCacheKey({
|
|
177
181
|
url: targetUrl,
|
|
178
182
|
options: {
|
|
@@ -182,19 +186,19 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
182
186
|
format: options.format,
|
|
183
187
|
markdownMode: options.markdownMode ?? null,
|
|
184
188
|
transcriptTimestamps: options.transcriptTimestamps ?? false,
|
|
185
|
-
...(typeof options.maxCharacters ===
|
|
189
|
+
...(typeof options.maxCharacters === "number"
|
|
186
190
|
? { maxCharacters: options.maxCharacters }
|
|
187
191
|
: {}),
|
|
188
192
|
},
|
|
189
193
|
})
|
|
190
194
|
: null;
|
|
191
195
|
if (!bypassExtractCache && cacheKey && cacheStore) {
|
|
192
|
-
const cached = cacheStore.getJson(
|
|
196
|
+
const cached = cacheStore.getJson("extract", cacheKey);
|
|
193
197
|
if (cached) {
|
|
194
|
-
writeVerbose(io.stderr, flags.verbose,
|
|
198
|
+
writeVerbose(io.stderr, flags.verbose, "cache hit extract", flags.verboseColor, io.envForRun);
|
|
195
199
|
return cached;
|
|
196
200
|
}
|
|
197
|
-
writeVerbose(io.stderr, flags.verbose,
|
|
201
|
+
writeVerbose(io.stderr, flags.verbose, "cache miss extract", flags.verboseColor, io.envForRun);
|
|
198
202
|
}
|
|
199
203
|
try {
|
|
200
204
|
const extracted = await fetchLinkContentWithBirdTip({
|
|
@@ -204,13 +208,13 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
204
208
|
env: io.env,
|
|
205
209
|
});
|
|
206
210
|
if (cacheKey && cacheStore) {
|
|
207
|
-
cacheStore.setJson(
|
|
208
|
-
writeVerbose(io.stderr, flags.verbose,
|
|
211
|
+
cacheStore.setJson("extract", cacheKey, extracted, cacheState.ttlMs);
|
|
212
|
+
writeVerbose(io.stderr, flags.verbose, "cache write extract", flags.verboseColor, io.envForRun);
|
|
209
213
|
}
|
|
210
214
|
return extracted;
|
|
211
215
|
}
|
|
212
216
|
catch (err) {
|
|
213
|
-
const preferUrlMode = typeof urlUtils.shouldPreferUrlMode ===
|
|
217
|
+
const preferUrlMode = typeof urlUtils.shouldPreferUrlMode === "function"
|
|
214
218
|
? urlUtils.shouldPreferUrlMode(targetUrl)
|
|
215
219
|
: false;
|
|
216
220
|
const isTwitter = urlUtils.isTwitterStatusUrl?.(targetUrl) ?? false;
|
|
@@ -219,7 +223,7 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
219
223
|
// Fallback: skip HTML fetch and proceed with URL-only extraction (YouTube/direct media).
|
|
220
224
|
writeVerbose(io.stderr, flags.verbose, `extract fallback url-only (${err.message ?? String(err)})`, flags.verboseColor, io.envForRun);
|
|
221
225
|
return {
|
|
222
|
-
content:
|
|
226
|
+
content: "",
|
|
223
227
|
title: null,
|
|
224
228
|
description: null,
|
|
225
229
|
url: targetUrl,
|
|
@@ -239,23 +243,23 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
239
243
|
transcriptTimedText: null,
|
|
240
244
|
transcriptionProvider: null,
|
|
241
245
|
diagnostics: {
|
|
242
|
-
strategy:
|
|
246
|
+
strategy: "html",
|
|
243
247
|
firecrawl: {
|
|
244
248
|
attempted: false,
|
|
245
249
|
used: false,
|
|
246
250
|
cacheMode: cacheState.mode,
|
|
247
|
-
cacheStatus:
|
|
248
|
-
notes:
|
|
251
|
+
cacheStatus: "bypassed",
|
|
252
|
+
notes: "skipped (url-only fallback)",
|
|
249
253
|
},
|
|
250
254
|
markdown: {
|
|
251
255
|
requested: false,
|
|
252
256
|
used: false,
|
|
253
257
|
provider: null,
|
|
254
|
-
notes:
|
|
258
|
+
notes: "skipped (url fallback)",
|
|
255
259
|
},
|
|
256
260
|
transcript: {
|
|
257
261
|
cacheMode: cacheState.mode,
|
|
258
|
-
cacheStatus:
|
|
262
|
+
cacheStatus: "unknown",
|
|
259
263
|
textProvided: false,
|
|
260
264
|
provider: null,
|
|
261
265
|
attemptedProviders: [],
|
|
@@ -270,7 +274,7 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
270
274
|
if (isTwitter) {
|
|
271
275
|
const refreshed = await fetchWithCache(url, { bypassExtractCache: true });
|
|
272
276
|
if (resolveSlideSource({ url, extracted: refreshed })) {
|
|
273
|
-
writeVerbose(io.stderr, flags.verbose,
|
|
277
|
+
writeVerbose(io.stderr, flags.verbose, "extract refresh for slides", flags.verboseColor, io.envForRun);
|
|
274
278
|
extracted = refreshed;
|
|
275
279
|
}
|
|
276
280
|
}
|
|
@@ -298,7 +302,7 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
298
302
|
extracted,
|
|
299
303
|
slides: null,
|
|
300
304
|
enabled: slidesOutputEnabled,
|
|
301
|
-
outputMode:
|
|
305
|
+
outputMode: "delta",
|
|
302
306
|
clearProgressForStdout: hooks.clearProgressForStdout,
|
|
303
307
|
restoreProgressAfterStdout: hooks.restoreProgressAfterStdout ?? null,
|
|
304
308
|
onProgressText: flags.progressEnabled
|
|
@@ -340,43 +344,44 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
340
344
|
try {
|
|
341
345
|
const source = resolveSlideSource({ url, extracted });
|
|
342
346
|
if (!source) {
|
|
343
|
-
throw new Error(
|
|
347
|
+
throw new Error("Slides are only supported for YouTube or direct video URLs.");
|
|
344
348
|
}
|
|
345
|
-
const slidesCacheKey = cacheStore && cacheState.mode ===
|
|
349
|
+
const slidesCacheKey = cacheStore && cacheState.mode === "default"
|
|
346
350
|
? buildSlidesCacheKey({ url: source.url, settings: flags.slides })
|
|
347
351
|
: null;
|
|
348
352
|
if (slidesCacheKey && cacheStore) {
|
|
349
|
-
const cached = cacheStore.getJson(
|
|
353
|
+
const cached = cacheStore.getJson("slides", slidesCacheKey);
|
|
350
354
|
const validated = cached
|
|
351
355
|
? await validateSlidesCache({ cached, source, settings: flags.slides })
|
|
352
356
|
: null;
|
|
353
357
|
if (validated) {
|
|
354
|
-
writeVerbose(io.stderr, flags.verbose,
|
|
358
|
+
writeVerbose(io.stderr, flags.verbose, "cache hit slides", flags.verboseColor, io.envForRun);
|
|
355
359
|
slidesExtracted = validated;
|
|
356
360
|
resolveTimeline(validated);
|
|
357
361
|
ctx.hooks.onSlidesExtracted?.(slidesExtracted);
|
|
358
|
-
ctx.hooks.onSlidesProgress?.(
|
|
362
|
+
ctx.hooks.onSlidesProgress?.("Slides: cached 100%");
|
|
359
363
|
return slidesExtracted;
|
|
360
364
|
}
|
|
361
|
-
writeVerbose(io.stderr, flags.verbose,
|
|
365
|
+
writeVerbose(io.stderr, flags.verbose, "cache miss slides", flags.verboseColor, io.envForRun);
|
|
362
366
|
}
|
|
363
367
|
if (flags.progressEnabled) {
|
|
364
|
-
spinner.setText(renderStatus(
|
|
365
|
-
oscProgress.setIndeterminate(
|
|
368
|
+
spinner.setText(renderStatus("Extracting slides"));
|
|
369
|
+
oscProgress.setIndeterminate("Extracting slides");
|
|
366
370
|
}
|
|
367
371
|
// Prefer indeterminate progress until we get real percentage updates from the slide pipeline.
|
|
368
|
-
ctx.hooks.onSlidesProgress?.(
|
|
372
|
+
ctx.hooks.onSlidesProgress?.("Slides: extracting");
|
|
369
373
|
const onSlidesLog = (message) => {
|
|
370
374
|
writeVerbose(io.stderr, flags.verbose, `slides ${message}`, flags.verboseColor, io.envForRun);
|
|
371
375
|
};
|
|
372
376
|
slidesExtracted = await extractSlidesForSource({
|
|
373
377
|
source,
|
|
374
378
|
settings: flags.slides,
|
|
375
|
-
noCache: cacheState.mode ===
|
|
379
|
+
noCache: cacheState.mode === "bypass",
|
|
376
380
|
mediaCache: ctx.mediaCache,
|
|
377
381
|
env: io.env,
|
|
378
382
|
timeoutMs: flags.timeoutMs,
|
|
379
383
|
ytDlpPath: model.apiStatus.ytDlpPath,
|
|
384
|
+
ytDlpCookiesFromBrowser: model.apiStatus.ytDlpCookiesFromBrowser,
|
|
380
385
|
ffmpegPath: null,
|
|
381
386
|
tesseractPath: null,
|
|
382
387
|
hooks: {
|
|
@@ -393,8 +398,8 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
393
398
|
ctx.hooks.onSlidesExtracted?.(slidesExtracted);
|
|
394
399
|
ctx.hooks.onSlidesProgress?.(`Slides: done (${slidesExtracted.slides.length.toString()} slides) 100%`);
|
|
395
400
|
if (slidesCacheKey && cacheStore) {
|
|
396
|
-
cacheStore.setJson(
|
|
397
|
-
writeVerbose(io.stderr, flags.verbose,
|
|
401
|
+
cacheStore.setJson("slides", slidesCacheKey, slidesExtracted, cacheState.ttlMs);
|
|
402
|
+
writeVerbose(io.stderr, flags.verbose, "cache write slides", flags.verboseColor, io.envForRun);
|
|
398
403
|
}
|
|
399
404
|
}
|
|
400
405
|
if (flags.progressEnabled) {
|
|
@@ -418,20 +423,20 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
418
423
|
const formatSummaryProgress = (modelId) => {
|
|
419
424
|
const dim = (value) => theme.dim(value);
|
|
420
425
|
const accent = (value) => theme.accent(value);
|
|
421
|
-
const sentLabel = `${dim(
|
|
422
|
-
const modelLabel = modelId ? `${dim(
|
|
423
|
-
const meta = modelLabel ? `${sentLabel}${dim(
|
|
424
|
-
return `${styleLabel(
|
|
426
|
+
const sentLabel = `${dim("sent ")}${extractionUi.contentSizeLabel}${extractionUi.viaSourceLabel}`;
|
|
427
|
+
const modelLabel = modelId ? `${dim("model: ")}${accent(modelId)}` : "";
|
|
428
|
+
const meta = modelLabel ? `${sentLabel}${dim(", ")}${modelLabel}` : sentLabel;
|
|
429
|
+
return `${styleLabel("Summarizing")} ${dim("(")}${meta}${dim(")")}${dim("…")}`;
|
|
425
430
|
};
|
|
426
431
|
const updateSummaryProgress = () => {
|
|
427
432
|
if (!flags.progressEnabled)
|
|
428
433
|
return;
|
|
429
434
|
websiteProgress?.stop?.();
|
|
430
435
|
if (!flags.extractMode) {
|
|
431
|
-
oscProgress.setIndeterminate(
|
|
436
|
+
oscProgress.setIndeterminate("Summarizing");
|
|
432
437
|
}
|
|
433
438
|
spinner.setText(flags.extractMode
|
|
434
|
-
? `${styleLabel(
|
|
439
|
+
? `${styleLabel("Extracted")}${styleDim(` (${extractionUi.contentSizeLabel}${extractionUi.viaSourceLabel})`)}`
|
|
435
440
|
: formatSummaryProgress());
|
|
436
441
|
};
|
|
437
442
|
updateSummaryProgress();
|
|
@@ -443,40 +448,40 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
443
448
|
env: io.envForRun,
|
|
444
449
|
});
|
|
445
450
|
const transcriptCacheStatus = extracted.diagnostics?.transcript?.cacheStatus;
|
|
446
|
-
if (transcriptCacheStatus && transcriptCacheStatus !==
|
|
451
|
+
if (transcriptCacheStatus && transcriptCacheStatus !== "unknown") {
|
|
447
452
|
writeVerbose(io.stderr, flags.verbose, `cache ${transcriptCacheStatus} transcript`, flags.verboseColor, io.envForRun);
|
|
448
453
|
}
|
|
449
454
|
if (flags.extractMode &&
|
|
450
455
|
markdown.markdownRequested &&
|
|
451
|
-
flags.preprocessMode !==
|
|
452
|
-
markdown.effectiveMarkdownMode ===
|
|
456
|
+
flags.preprocessMode !== "off" &&
|
|
457
|
+
markdown.effectiveMarkdownMode === "auto" &&
|
|
453
458
|
!extracted.diagnostics.markdown.used &&
|
|
454
459
|
!hasUvxCli(io.env)) {
|
|
455
460
|
io.stderr.write(`${UVX_TIP}\n`);
|
|
456
461
|
}
|
|
457
462
|
if (!isYoutubeUrl && extracted.isVideoOnly && extracted.video) {
|
|
458
|
-
if (extracted.video.kind ===
|
|
463
|
+
if (extracted.video.kind === "youtube") {
|
|
459
464
|
writeVerbose(io.stderr, flags.verbose, `video-only page detected; switching to YouTube URL ${extracted.video.url}`, flags.verboseColor, io.envForRun);
|
|
460
465
|
if (flags.progressEnabled) {
|
|
461
|
-
spinner.setText(renderStatus(
|
|
466
|
+
spinner.setText(renderStatus("Video-only page", ": fetching YouTube transcript…"));
|
|
462
467
|
}
|
|
463
468
|
extracted = await fetchWithCache(extracted.video.url);
|
|
464
469
|
extractionUi = deriveExtractionUi(extracted);
|
|
465
470
|
updateSummaryProgress();
|
|
466
471
|
}
|
|
467
|
-
else if (extracted.video.kind ===
|
|
472
|
+
else if (extracted.video.kind === "direct") {
|
|
468
473
|
const directVideoSlides = await runSlidesExtraction();
|
|
469
|
-
const wantsVideoUnderstanding = flags.videoMode ===
|
|
474
|
+
const wantsVideoUnderstanding = flags.videoMode === "understand" || flags.videoMode === "auto";
|
|
470
475
|
// Direct video URLs require a model that can consume video attachments (currently Gemini).
|
|
471
476
|
const canVideoUnderstand = wantsVideoUnderstanding &&
|
|
472
477
|
model.apiStatus.googleConfigured &&
|
|
473
|
-
(model.requestedModel.kind ===
|
|
474
|
-
(model.fixedModelSpec?.transport ===
|
|
475
|
-
model.fixedModelSpec.provider ===
|
|
478
|
+
(model.requestedModel.kind === "auto" ||
|
|
479
|
+
(model.fixedModelSpec?.transport === "native" &&
|
|
480
|
+
model.fixedModelSpec.provider === "google"));
|
|
476
481
|
if (canVideoUnderstand) {
|
|
477
482
|
hooks.onExtracted?.(extracted);
|
|
478
483
|
if (flags.progressEnabled)
|
|
479
|
-
spinner.setText(renderStatus(
|
|
484
|
+
spinner.setText(renderStatus("Downloading video"));
|
|
480
485
|
const loadedVideo = await loadRemoteAsset({
|
|
481
486
|
url: extracted.video.url,
|
|
482
487
|
fetchImpl: io.fetch,
|
|
@@ -485,17 +490,17 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
485
490
|
assertAssetMediaTypeSupported({ attachment: loadedVideo.attachment, sizeLabel: null });
|
|
486
491
|
let chosenModel = null;
|
|
487
492
|
if (flags.progressEnabled)
|
|
488
|
-
spinner.setText(renderStatus(
|
|
493
|
+
spinner.setText(renderStatus("Summarizing video"));
|
|
489
494
|
await hooks.summarizeAsset({
|
|
490
|
-
sourceKind:
|
|
495
|
+
sourceKind: "asset-url",
|
|
491
496
|
sourceLabel: loadedVideo.sourceLabel,
|
|
492
497
|
attachment: loadedVideo.attachment,
|
|
493
498
|
onModelChosen: (modelId) => {
|
|
494
499
|
chosenModel = modelId;
|
|
495
500
|
hooks.onModelChosen?.(modelId);
|
|
496
501
|
if (flags.progressEnabled) {
|
|
497
|
-
const meta = `${styleDim(
|
|
498
|
-
spinner.setText(renderStatusWithMeta(
|
|
502
|
+
const meta = `${styleDim("(")}${styleDim("model: ")}${theme.accent(modelId)}${styleDim(")")}`;
|
|
503
|
+
spinner.setText(renderStatusWithMeta("Summarizing video", meta));
|
|
499
504
|
}
|
|
500
505
|
},
|
|
501
506
|
});
|
|
@@ -538,20 +543,21 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
538
543
|
mediaDurationSeconds: extracted.mediaDurationSeconds,
|
|
539
544
|
openaiWhisperUsdPerMinute: model.openaiWhisperUsdPerMinute,
|
|
540
545
|
});
|
|
541
|
-
const transcriptionCostLabel = typeof transcriptionCostUsd ===
|
|
546
|
+
const transcriptionCostLabel = typeof transcriptionCostUsd === "number" ? `txcost=${formatUSD(transcriptionCostUsd)}` : null;
|
|
542
547
|
hooks.setTranscriptionCost(transcriptionCostUsd, transcriptionCostLabel);
|
|
543
548
|
if (flags.extractMode) {
|
|
544
549
|
// Apply transcript→markdown conversion if requested
|
|
545
550
|
let extractedForOutput = extracted;
|
|
546
551
|
if (markdown.transcriptMarkdownRequested && markdown.convertTranscriptToMarkdown) {
|
|
547
552
|
if (flags.progressEnabled) {
|
|
548
|
-
spinner.setText(renderStatus(
|
|
553
|
+
spinner.setText(renderStatus("Converting transcript to markdown"));
|
|
549
554
|
}
|
|
550
555
|
const markdownContent = await markdown.convertTranscriptToMarkdown({
|
|
551
556
|
title: extracted.title,
|
|
552
557
|
source: extracted.siteName,
|
|
553
558
|
transcript: extracted.content,
|
|
554
559
|
timeoutMs: flags.timeoutMs,
|
|
560
|
+
outputLanguage: flags.outputLanguage,
|
|
555
561
|
});
|
|
556
562
|
extractedForOutput = {
|
|
557
563
|
...extracted,
|
|
@@ -562,8 +568,8 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
562
568
|
...extracted.diagnostics.markdown,
|
|
563
569
|
requested: true,
|
|
564
570
|
used: true,
|
|
565
|
-
provider:
|
|
566
|
-
notes:
|
|
571
|
+
provider: "llm",
|
|
572
|
+
notes: "transcript",
|
|
567
573
|
},
|
|
568
574
|
},
|
|
569
575
|
};
|
|
@@ -603,8 +609,8 @@ export async function runUrlFlow({ ctx, url, isYoutubeUrl, }) {
|
|
|
603
609
|
}
|
|
604
610
|
finally {
|
|
605
611
|
if (flags.progressEnabled) {
|
|
606
|
-
process.off(
|
|
607
|
-
process.off(
|
|
612
|
+
process.off("SIGINT", handleSigint);
|
|
613
|
+
process.off("SIGTERM", handleSigterm);
|
|
608
614
|
}
|
|
609
615
|
hooks.clearProgressIfCurrent(pauseProgressLine);
|
|
610
616
|
stopProgress();
|