@signal4/s4 2.7.0 → 2.8.0

package/README.md

@@ -38,6 +38,11 @@ s4 count tariffs --since 2025 --by month     # segments · episodes · channels
 s4 count "carbon tax" -m keyword --collection Canadian --by channel   # exact; rank shows by what they said
 s4 count "carbon tax" --vs "carbon pricing" --since 2024 --by quarter # several topics side by side
 s4 count "Strong Borders Act" -m keyword --by episode --order date -n 100  # every matching episode, newest first
+s4 count "carbon tax" -m keyword --since 2025 --by speaker   # who speaks in the matching segments, names corroborated per episode
+s4 top-speakers --topic tariffs -d 90       # same ranking, from the transcripts (not episode titles)
+s4 entity-claims "Donald Trump"             # every entity row of that name + merges; says what it matched
+s4 search tariffs --keyword --since 2025 --page 3   # pages of -n; --all follows every page
+s4 top-channels --topic Medicaid --country us -d 365 # channels ranked by what they said
 s4 segment 42314442                         # full text of a quote (seg:ID from search)
 s4 --version
 ```
@@ -45,9 +50,31 @@ s4 --version
 Flags: `-n/-k/--limit` results · `--offset` skip · `-s` speaker · `-c` channel · `-d/--days` · `-m` mode ·
 `-o/--out` file · `--since/--until` (or `--start/--end`; `2023`, `2023-06`, `2023-06-15`; on search, claims, timeline, trend, the claims-analysis commands, top-claims, top-channels, top-speakers) · `--collection` (every corpus command) · `--json`.
 `--days` has no ceiling on search, claims, timeline and count; report commands cap at 365.
-`count` groups with `--by` (day · week · month · quarter · year · channel · episode · language · country · platform · collection · lean);
+`count` groups with `--by` (day · week · month · quarter · year · channel · episode · speaker · language · country · platform · collection · lean);
 keyword counts are exact, semantic counts are estimates at similarity ≥0.35, and every language is counted unless `--language` names some. Errors go to stderr with exit 1.
 
+## Paging
+
+Every list command and tool pages the same way: `limit` + `offset` (CLI `-n` and `--offset`,
+or `--page N`), and every answer ends with the next call (`next: offset:40`, CLI `--offset 40`).
+`s4 <command> --all` follows the pages for you (up to 50). How deep each list goes:
+
+| List | Depth |
+|---|---|
+| keyword search, `count` breakdowns, entity claims, claim observations | every match |
+| semantic / hybrid quote search | rank 2,000 (`depth_reached` says when you hit it) |
+| semantic / hybrid claims search | rank 1,000 |
+| top speakers / channels / talk time | 200 (topic rankings: every match) |
+| top claims | 100 |
+
+Past a semantic depth, `--keyword` search or `count ... --by episode` list every match.
+From TypeScript, `paginate()` walks the pages of any list call:
+
+```typescript
+import { paginate, searchQuotes } from '@signal4/s4/client';
+for await (const quote of paginate(searchQuotes, { topic: 'tariffs', mode: 'keyword', days: 365 })) { /* ... */ }
+```
+
 ## Speed
 
 Every MCP tool description and every `s4 <command> --help` starts with how long the call takes at

package/dist/cli.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAgnD7E,wBAAsB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAmG1C"}
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA,6EAA6E;AA0qD7E,wBAAsB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CA+G1C"}

package/dist/cli.js

@@ -5,7 +5,7 @@ import { descriptionTag, estimate, startNotice, REPORT_ELAPSED_OVER_S } from "./
 import * as client from "./client.js";
 import { writeFileSync, mkdirSync, existsSync, chmodSync, readFileSync, statSync } from "node:fs";
 import { join } from "node:path";
-import { formatDossier } from "./format.js";
+import { formatDossier, parsePoliticalLean } from "./format.js";
 import { homedir } from "node:os";
 import { createInterface } from "node:readline/promises";
 import { parseArgs as parseArgsRaw, UsageError, didYouMean } from "./cli_args.js";
@@ -29,6 +29,12 @@ function parseArgs(argv, specs) {
 const FLAG_DAYS = { name: "days", alias: "d", type: "number", default: 30, desc: "Time window in days (report commands cap at 365; search/claims/timeline do not)" };
 const FLAG_LIMIT = { name: "limit", alias: ["n", "k"], type: "number", default: 20, desc: "Number of results (-n, -k)" };
 const FLAG_OFFSET = { name: "offset", type: "number", default: 0, desc: "Skip this many results (pagination)" };
+const FLAG_PAGE = { name: "page", type: "number", desc: "Page number, 1-based, in pages of --limit (sets --offset)" };
+const FLAG_ALL = { name: "all", type: "boolean", desc: `Follow every page (up to ${50} pages)` };
+/** Paging flags for every list command (signal4#329). */
+const PAGING = [FLAG_OFFSET, FLAG_PAGE, FLAG_ALL];
+/** --all stops after this many pages; the last page then says how to continue. */
+const MAX_ALL_PAGES = 50;
 const FLAG_SPEAKER = { name: "speaker", alias: "s", type: "string", desc: "Filter by speaker name (full name)" };
 const FLAG_CHANNEL = { name: "channel", alias: "c", type: "string", desc: "Filter by channel name (comma-separated for multiple)" };
 const FLAG_LANGUAGE = { name: "language", type: "string", desc: "Filter by language code (en, fr, ...)" };
@@ -42,7 +48,7 @@ const FLAG_SKIP_CACHE = { name: "skip-cache", type: "boolean", desc: "Force fres
 const FLAG_OUT = { name: "out", alias: "o", long: ["output"], type: "string", desc: "Write the rendered output to a file (.md, or .csv on search)" };
 const FLAG_JSON = { name: "json", type: "boolean", desc: "Raw JSON output" };
 const GRANULARITIES = ["day", "week", "month", "quarter", "year"];
-const COUNT_GROUPS = ["none", ...GRANULARITIES, "channel", "episode", "language", "country", "platform", "collection", "lean"];
+const COUNT_GROUPS = ["none", ...GRANULARITIES, "channel", "episode", "speaker", "language", "country", "platform", "collection", "lean"];
 /** --start/--end as tool params (already normalized to YYYY-MM-DD by parseArgs); absent when not given. */
 function dateFlags(f) {
     const out = {};
@@ -217,12 +223,40 @@ async function runHandler(tool, params, out) {
         process.stderr.write((result.content[0]?.text ?? "Unknown error") + "\n");
         process.exit(1);
     }
-    const text = result.content[0]?.text ?? "";
+    let text = result.content[0]?.text ?? "";
+    // --all: follow the next-page hint every list answer ends with, until it stops.
+    let pages = 1;
+    let next = nextPage(text);
+    while (followAllPages && next && pages < MAX_ALL_PAGES) {
+        const current = Number(params[next.param] ?? 0);
+        if (next.offset <= current)
+            break;
+        params = { ...params, [next.param]: next.offset };
+        const more = await handler(params);
+        if (more.isError)
+            break;
+        const pageText = more.content[0]?.text ?? "";
+        text += `\n\n--- page ${++pages} (${next.param} ${next.offset}) ---\n` + pageText;
+        next = nextPage(pageText);
+    }
+    if (followAllPages && next && pages >= MAX_ALL_PAGES) {
+        text += `\n\n(--all stopped after ${MAX_ALL_PAGES} pages; continue with --offset ${next.offset})`;
+    }
     if (out)
         writeOut(out, text);
     else
         print(text);
 }
+/** Set by main from --all. */
+let followAllPages = false;
+/** The next page a list answer offers ("--offset 40", "--obs-offset 40"), or null. */
+function nextPage(text) {
+    const obs = [...text.matchAll(/--obs-offset (\d+)/g)].pop();
+    if (obs)
+        return { param: "observations_offset", offset: Number(obs[1]) };
+    const m = [...text.matchAll(/(?<![-\w])--offset (\d+)/g)].pop();
+    return m ? { param: "offset", offset: Number(m[1]) } : null;
+}
 // A `--channel` name that doesn't resolve to a real channel must fail loudly
 // in keyword mode, not fall through as an unfiltered corpus-wide query
 // (signal4#85). Keyword-mode search silently drops channel_names server-side
@@ -411,7 +445,7 @@ const COMMANDS = {
     search: {
         tool: "search_quotes",
         description: "Search quotes by topic and/or speaker",
-        specs: [FLAG_SPEAKER, FLAG_CHANNEL, FLAG_DAYS, FLAG_MODE, { ...FLAG_LIMIT, default: 10 }, FLAG_OFFSET, FLAG_LANGUAGE, FLAG_COUNTRY,
+        specs: [FLAG_SPEAKER, FLAG_CHANNEL, FLAG_DAYS, FLAG_MODE, { ...FLAG_LIMIT, default: 10 }, ...PAGING, FLAG_LANGUAGE, FLAG_COUNTRY,
             { name: "sort", type: "string", default: "relevance", desc: "relevance | chronological" },
             { name: "count", type: "boolean", desc: "Return only the match count" },
             { name: "full-history", type: "boolean", desc: "Search the entire corpus (requires topic + speaker)" },
@@ -500,7 +534,7 @@ const COMMANDS = {
             { name: "political-lean-min", type: "number", desc: "Min channel political lean (0=left, 1=right)" },
             { name: "political-lean-max", type: "number", desc: "Max channel political lean (0=left, 1=right)" },
             { name: "source", type: "string", desc: "Claim source filter" },
-            FLAG_DAYS, { ...FLAG_LIMIT, default: 10 }, FLAG_OFFSET, FLAG_START, FLAG_END, FLAG_COLLECTION, FLAG_OUT, FLAG_JSON],
+            FLAG_DAYS, { ...FLAG_LIMIT, default: 10 }, ...PAGING, FLAG_START, FLAG_END, FLAG_COLLECTION, FLAG_OUT, FLAG_JSON],
         run: async (args, json) => {
             const [positionalTopic] = args.positionals;
             const f = args.flags;
@@ -733,8 +767,8 @@ const COMMANDS = {
     },
     "entities": {
         tool: "find_entities",
-        description: "Fuzzy search canonical entities by name/alias (people, orgs, places, subjects)",
-        specs: [{ name: "type", type: "string", desc: "Entity type filter: person | organization | place | subject" }, FLAG_LIMIT, FLAG_JSON],
+        description: "Find entities by name (people, orgs, places, subjects); the first row is what entity-claims uses",
+        specs: [{ name: "type", type: "string", desc: "Entity type filter: person | organization | place | subject" }, FLAG_LIMIT, ...PAGING, FLAG_JSON],
         run: async (args, json) => {
             const [query] = args.positionals;
             if (!query) {
@@ -744,17 +778,18 @@ const COMMANDS = {
             const f = args.flags;
             const entityType = f.type || undefined;
             const limit = Number(f.limit) || undefined;
+            const offset = Number(f.offset) || undefined;
             if (json)
-                return printJson(await client.searchEntities({ query, entity_type: entityType, limit }));
-            await runHandler("find_entities", { query, entity_type: entityType, limit });
+                return printJson(await client.searchEntities({ query, entity_type: entityType, limit, offset }));
+            await runHandler("find_entities", { query, entity_type: entityType, limit, offset });
         },
     },
     "entity-claims": {
         tool: "entity_claims",
-        description: "Claims about a resolved canonical entity, ranked, with stance distribution",
+        description: "Most-spread claims about an entity (every row of that name or alias, merges included)",
         specs: [{ name: "sort", type: "string", default: "spread", desc: "Sort: spread | observations | recent" },
             { name: "stance", type: "string", desc: "Filter by stance: assert | deny | speculate | attribute" },
-            FLAG_LIMIT, FLAG_OFFSET, FLAG_COLLECTION, FLAG_JSON],
+            FLAG_LIMIT, ...PAGING, FLAG_COLLECTION, FLAG_JSON],
         run: async (args, json) => {
             const [name] = args.positionals;
             if (!name) {
@@ -796,13 +831,14 @@ const COMMANDS = {
         specs: [{ name: "metric", type: "string", default: "spread", desc: "Ranking: spread (reach) | contested (assert vs deny balance)" },
             { name: "category", type: "string", desc: "Optional topic-category filter" },
             { name: "min-obs", type: "number", default: 2, desc: "Minimum observation count per claim" }, FLAG_DAYS, FLAG_START, FLAG_END,
-            FLAG_LANGUAGE, FLAG_COLLECTION, FLAG_LIMIT, FLAG_JSON],
+            FLAG_LANGUAGE, FLAG_COLLECTION, FLAG_LIMIT, ...PAGING, FLAG_JSON],
         run: async (args, json) => {
             const f = args.flags;
             const params = {
                 metric: f.metric || undefined, category: f.category || undefined,
                 days: Number(f.days) || undefined, min_observations: Number(f["min-obs"]) || undefined,
-                limit: Number(f.limit) || undefined, language: f.language || undefined,
+                limit: Number(f.limit) || undefined, offset: Number(f.offset) || undefined,
+                language: f.language || undefined,
                 ...dateFlags(f),
             };
             if (json)
@@ -813,7 +849,10 @@ const COMMANDS = {
     claim: {
         tool: "get_claim",
         description: "Get full claim detail with observations and related claims",
-        specs: [{ name: "observations", type: "boolean", desc: "Include the full observation list" },
+        specs: [{ name: "observations", type: "boolean", desc: "Include the observation list (20 a page)" },
+            { name: "obs-offset", type: "number", desc: "Observations: skip this many (implies --observations)" },
+            { name: "obs-limit", type: "number", desc: "Observations per page (max 100)" },
+            FLAG_ALL,
             { name: "related", type: "boolean", desc: "Include semantically related claims" }, FLAG_JSON],
         run: async (args, json) => {
             const [id] = args.positionals;
@@ -822,17 +861,22 @@ const COMMANDS = {
                 process.exit(1);
             }
             const claimId = Number(id);
+            const obsOffset = args.flags["obs-offset"] != null ? Number(args.flags["obs-offset"]) : undefined;
+            const obsLimit = args.flags["obs-limit"] != null ? Number(args.flags["obs-limit"]) : undefined;
             if (json) {
                 const detail = await client.getClaim(claimId);
-                if (args.flags.observations)
-                    detail.observations = await client.getClaimObservations({ claim_id: claimId });
+                if (args.flags.observations || obsOffset != null || obsLimit != null) {
+                    detail.observations = await client.getClaimObservations({ claim_id: claimId, limit: obsLimit, offset: obsOffset });
+                }
                 if (args.flags.related)
                     detail.related = await client.getRelatedClaims(claimId);
                 return printJson(detail);
             }
             await runHandler("get_claim", {
                 claim_id: claimId,
-                include_observations: args.flags.observations,
+                include_observations: args.flags.observations || followAllPages,
+                observations_offset: obsOffset,
+                observations_limit: obsLimit,
                 include_related: args.flags.related,
             });
         },
@@ -933,7 +977,7 @@ const COMMANDS = {
             { name: "vs", type: "string", desc: "More topics to count side by side, comma-separated (up to 5 in all)" },
             { name: "order", type: "string", default: "segments", desc: "Episode breakdown order: segments | date" },
             { name: "min-similarity", type: "number", desc: "Semantic threshold (default 0.35)" },
-            { ...FLAG_LIMIT, desc: "Rows for channel/episode/... breakdowns (-n, -k; max 1000)" }, FLAG_OFFSET, FLAG_JSON],
+            { ...FLAG_LIMIT, desc: "Rows for channel/episode/... breakdowns (-n, -k; max 1000)" }, ...PAGING, FLAG_JSON],
         run: async (args, json) => {
             const f = args.flags;
             const topic = args.positionals.join(" ").trim() || undefined;
@@ -971,7 +1015,7 @@ const COMMANDS = {
     speaker: {
         tool: "find_speakers",
         description: "Speaker search, profile, or discourse",
-        specs: [FLAG_DAYS, FLAG_LIMIT, FLAG_OFFSET, FLAG_CHANNEL,
+        specs: [FLAG_DAYS, FLAG_LIMIT, ...PAGING, FLAG_CHANNEL,
             { name: "segments", type: "boolean", desc: "List the speaker's recent segments (chronological)" }, FLAG_JSON],
         run: async (args, json) => {
             const [identity, topic] = args.positionals;
@@ -1045,7 +1089,7 @@ const COMMANDS = {
     channel: {
         tool: "get_channel",
         description: "Channel profile or episode listing",
-        specs: [{ name: "episodes", type: "boolean", desc: "List the channel's recent episodes instead of the profile" }, FLAG_DAYS, FLAG_LIMIT, FLAG_OFFSET, FLAG_JSON],
+        specs: [{ name: "episodes", type: "boolean", desc: "List the channel's recent episodes instead of the profile" }, FLAG_DAYS, FLAG_LIMIT, ...PAGING, FLAG_JSON],
         run: async (args, json) => {
             const [id] = args.positionals;
             if (!id) {
@@ -1378,26 +1422,38 @@ const COMMANDS = {
     },
     "top-channels": {
         tool: "get_top_channels",
-        description: "Top channels by episode count",
-        specs: [FLAG_DAYS, FLAG_START, FLAG_END, FLAG_LIMIT, FLAG_LANGUAGE, FLAG_COUNTRY, FLAG_COLLECTION, FLAG_JSON],
+        description: "Top channels by episode count; with --topic, by how much they say about it",
+        specs: [FLAG_DAYS, FLAG_START, FLAG_END, FLAG_LIMIT, ...PAGING, FLAG_LANGUAGE, FLAG_COUNTRY, FLAG_COLLECTION,
+            { name: "topic", type: "string", desc: "Rank by matching segments per channel, from the transcripts" },
+            { name: "lean", type: "string", desc: "Channel lean: left | center-left | center | center-right | right | 0.0-0.5" },
+            FLAG_JSON],
         run: async (args, json) => {
             const f = args.flags;
-            const params = { days: Number(f.days) || 30, limit: Number(f.limit) || 20, language: f.language || undefined, country: f.country || undefined, ...dateFlags(f) };
-            if (json)
-                return printJson(await client.getTopChannels({ ...params, project: f.collection || undefined }));
+            const topic = f.topic || args.positionals.join(" ") || undefined;
+            const params = { days: Number(f.days) || 30, limit: Number(f.limit) || 20, offset: Number(f.offset) || undefined, language: f.language || undefined, country: f.country || undefined, topic, political_lean: f.lean || undefined, ...dateFlags(f) };
+            if (json) {
+                const lean = parsePoliticalLean(params.political_lean);
+                return printJson(await client.getTopChannels({ ...params, political_lean_min: lean.min, political_lean_max: lean.max, project: f.collection || undefined }));
+            }
             await runHandler("get_top_channels", params);
         },
     },
     "top-speakers": {
         tool: "get_top_speakers",
-        description: "Top speakers by appearances, talk time, or clips",
+        description: "Top speakers by appearances, talk time or clips; with --topic, by how much they talk about it",
         specs: [FLAG_DAYS, FLAG_START, FLAG_END, FLAG_LIMIT, FLAG_LANGUAGE, FLAG_COUNTRY, FLAG_COLLECTION,
-            { name: "ranking", type: "string", default: "appearances", desc: "Rank by: appearances | talkative (talk time) | clipped" }, FLAG_JSON],
+            { name: "ranking", type: "string", default: "appearances", desc: "Rank by: appearances | talkative (talk time) | clipped" },
+            { name: "topic", type: "string", desc: "Rank by matching segments each speaker speaks in, from the transcripts (overrides --ranking)" },
+            { name: "lean", type: "string", desc: "Channel lean: left | center-left | center | center-right | right | 0.0-0.5" },
+            ...PAGING, FLAG_JSON],
         run: async (args, json) => {
             const f = args.flags;
-            const params = { days: Number(f.days) || 30, limit: Number(f.limit) || 20, ranking: f.ranking, language: f.language || undefined, country: f.country || undefined, project: f.collection || undefined, ...dateFlags(f) };
-            if (json)
-                return printJson(await client.getTopSpeakers(params));
+            const topic = f.topic || args.positionals.join(" ") || undefined;
+            const params = { days: Number(f.days) || 30, limit: Number(f.limit) || 20, offset: Number(f.offset) || undefined, ranking: f.ranking, language: f.language || undefined, country: f.country || undefined, project: f.collection || undefined, topic, political_lean: f.lean || undefined, ...dateFlags(f) };
+            if (json) {
+                const lean = parsePoliticalLean(params.political_lean);
+                return printJson(await client.getTopSpeakers({ ...params, political_lean_min: lean.min, political_lean_max: lean.max }));
+            }
             await runHandler("get_top_speakers", params);
         },
     },
@@ -1438,7 +1494,7 @@ const COMMANDS = {
     ads: {
         tool: "get_ad_analytics",
         description: "Ad intelligence",
-        specs: [FLAG_DAYS, FLAG_LIMIT, FLAG_OFFSET,
+        specs: [FLAG_DAYS, FLAG_LIMIT, ...PAGING,
             { name: "advertisers", type: "boolean", desc: "List advertisers instead of the overview" },
             { name: "trending", type: "boolean", desc: "Show growing/declining/new advertisers" },
             FLAG_CHANNEL,
@@ -1496,7 +1552,7 @@ const COMMANDS = {
         description: "Manage bookmarks",
         specs: [{ name: "type", type: "string", desc: "Filter by bookmark type: segment | episode | speaker | channel" },
             { name: "search", type: "string", desc: "Filter bookmarks by search text" },
-            FLAG_LIMIT, FLAG_OFFSET,
+            FLAG_LIMIT, ...PAGING,
             { name: "note", type: "string", desc: "Note to attach when adding a bookmark" }, FLAG_JSON],
         run: async (args, json) => {
             const [sub, entityType, entityId] = args.positionals;
@@ -1814,6 +1870,20 @@ export async function main() {
     }
     const json = args.flags.json;
     activeCollection = args.flags.collection || undefined;
+    // --page N is --offset (N-1)*limit; --all follows the next-page hint (signal4#329).
+    const page = Number(args.flags.page);
+    if (page) {
+        if (page < 1 || !Number.isInteger(page)) {
+            printErr("Error: --page is a whole number from 1.");
+            process.exit(1);
+        }
+        args.flags.offset = (page - 1) * (Number(args.flags.limit) || 20);
+    }
+    followAllPages = !!args.flags.all;
+    if (followAllPages && json) {
+        printErr("Error: --all pages the text output. For JSON, page with --offset, or use paginate() from @signal4/s4/client.");
+        process.exit(1);
+    }
     // --days has no ceiling where the backend takes dates; report endpoints cap at 365.
     const days = args.flags.days;
     if (typeof days === "number" && days > 365 && !args.flags.start && !DATE_RANGE_COMMANDS.has(command)) {