@steipete/summarize 0.1.0

package/docs/website.md

@@ -0,0 +1,27 @@
+# Website mode
+
+Use this for non-YouTube URLs.
+
+## What it does
+
+- Fetches the page HTML.
+- Extracts “article-ish” content and normalizes it into clean text.
+- If extraction looks blocked or too thin, it can retry via Firecrawl (Markdown).
+- In `--extract-only` mode, the CLI prefers Firecrawl Markdown by default when `FIRECRAWL_API_KEY` is configured.
+- In `--extract-only` mode, `--markdown auto|llm` can also convert HTML → Markdown via an LLM using the configured `--model` (no provider fallback).
+
+## Flags
+
+- `--firecrawl off|auto|always`
+- `--markdown off|auto|llm` (default: `auto`; only affects `--extract-only` for non-YouTube URLs)
+- Plain-text mode: use `--firecrawl off --markdown off`.
+- `--timeout 30s|30|2m|5000ms` (default: `2m`)
+- `--extract-only` (print extracted content; no summary LLM call)
+- `--json` (emit a single JSON object)
+- `--verbose` (progress + which extractor was used)
+- `--metrics off|on|detailed` (default: `on`; `detailed` prints a breakdown to stderr)
+
+## API keys
+
+- Optional: `FIRECRAWL_API_KEY` (for the Firecrawl fallback / preferred Markdown output)
+- Optional: `XAI_API_KEY` / `OPENAI_API_KEY` / `GEMINI_API_KEY` (also accepts `GOOGLE_GENERATIVE_AI_API_KEY` / `GOOGLE_API_KEY`) (required only when `--markdown llm` is used, or when `--markdown auto` falls back to LLM conversion)

package/docs/youtube.md

@@ -0,0 +1,32 @@
+# YouTube mode
+
+YouTube URLs use transcript-first extraction.
+
+## `--youtube auto|web|apify`
+
+- `auto` (default): try `youtubei` → `captionTracks` → Apify (if token exists)
+- `web`: try `youtubei` → `captionTracks` only
+- `apify`: Apify only
+
+## `youtubei` vs `captionTracks`
+
+- `youtubei`:
+  - Calls YouTube’s internal transcript endpoint (`/youtubei/v1/get_transcript`).
+  - Needs a bootstrapped `INNERTUBE_API_KEY`, context, and `getTranscriptEndpoint.params` from the watch page HTML.
+  - When it works, you get a nice list of transcript segments.
+- `captionTracks`:
+  - Downloads caption tracks listed in `ytInitialPlayerResponse.captions.playerCaptionsTracklistRenderer.captionTracks`.
+  - Fetches `fmt=json3` first and falls back to XML-like caption payloads if needed.
+  - Often works even when the transcript endpoint doesn’t.
+
+## Fallbacks
+
+- If no transcript is available, we still extract `ytInitialPlayerResponse.videoDetails.shortDescription` so YouTube links can still summarize meaningfully.
+- Apify is an optional fallback (needs `APIFY_API_TOKEN`).
+  - By default, we use the actor id `faVsWy9VTSNVIhWpR` (Pinto Studio’s “Youtube Transcript Scraper”).
+
+## Example
+
+```bash
+pnpm summarize -- --extract-only "https://www.youtube.com/watch?v=I845O57ZSy4&t=11s"
+```

package/package.json

@@ -0,0 +1,76 @@
+{
+  "name": "@steipete/summarize",
+  "version": "0.1.0",
+  "description": "Link → clean text → summary.",
+  "type": "module",
+  "bin": {
+    "summarize": "./dist/cli.cjs"
+  },
+  "main": "./dist/esm/index.js",
+  "module": "./dist/esm/index.js",
+  "types": "./dist/types/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/esm/index.js"
+    },
+    "./content": {
+      "types": "./dist/types/content/index.d.ts",
+      "import": "./dist/esm/content/index.js"
+    },
+    "./prompts": {
+      "types": "./dist/types/prompts/index.d.ts",
+      "import": "./dist/esm/prompts/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "docs",
+    "CHANGELOG.md",
+    "README.md",
+    "LICENSE"
+  ],
+  "dependencies": {
+    "cheerio": "^1.1.2",
+    "es-toolkit": "^1.43.0",
+    "sanitize-html": "^2.17.0"
+  },
+  "devDependencies": {
+    "@ai-sdk/anthropic": "3.0.0-beta.91",
+    "@ai-sdk/google": "3.0.0-beta.81",
+    "@ai-sdk/openai": "3.0.0-beta.104",
+    "@ai-sdk/xai": "3.0.0-beta.62",
+    "@biomejs/biome": "^2.3.10",
+    "@types/node": "^25.0.3",
+    "@types/sanitize-html": "^2.16.0",
+    "@vitest/coverage-v8": "^4.0.16",
+    "ai": "6.0.0-beta.159",
+    "commander": "^14.0.2",
+    "esbuild": "^0.27.2",
+    "file-type": "^21.1.1",
+    "markdansi": "0.1.7",
+    "mime": "^4.1.0",
+    "osc-progress": "^0.1.0",
+    "ora": "^9.0.0",
+    "tokentally": "github:steipete/tokentally#v0.1.0",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.16"
+  },
+  "scripts": {
+    "build": "pnpm build:lib && pnpm build:cli",
+    "build:lib": "tsc -p tsconfig.build.json",
+    "build:cli": "node scripts/build-cli.mjs",
+    "build:bun": "bun scripts/build-bun.js",
+    "build:bun:test": "bun scripts/build-bun.js --test",
+    "typecheck": "tsc -p tsconfig.build.json --noEmit",
+    "summarize": "tsx src/cli.ts",
+    "format": "biome format --write .",
+    "lint": "biome check .",
+    "lint:fix": "biome check --write .",
+    "check": "pnpm lint && pnpm test:coverage",
+    "test": "pnpm build && vitest run",
+    "test:coverage": "pnpm build && vitest run --coverage",
+    "release": "bash scripts/release.sh"
+  }
+}