@finbheara/archive-client 0.4.0 → 0.5.0

package/README.md

@@ -20,7 +20,7 @@ type declarations (`.d.ts`), so a TypeScript project needs Bun's types (`"types"
 `bunx idq <verb>`.
 
 The version is 0.x while the API settles: a minor bump may break, so pin a minor
-(`"@finbheara/archive-client": "0.4.x"`).
+(`"@finbheara/archive-client": "0.5.x"`).
 
 ## Configuration
 
@@ -85,6 +85,11 @@ c.features;   // [{ feature: "dancers", state: "full" | "missing", missing: [...
 | `staff` | `staff.teacher`, `staff.adjudicator` | 260 |
 | `majors.order` | `majors.competitions` | 260 |
 | `feises`, `comps` | `feises.find`, `feises.series`, `comps.*`, `registrations.resolve` | 265 |
+| `results.stream`, `majors.stream` | `results.stream`, `results.entries` | 255 |
+| `feises.dates` | `feises.dates` | 245 |
+| `dancers.records` | `dancers.records` | 215 |
+| `dancers.years` | `dancers.years` | 245 |
+| `majors.tabs` | `majors.tabulated` | 215 |
 | `teams` | `teams.results`, `teams.rosters`, `teams.resultRosters` | 260 |
 | `events` | `events.years` | 260 |
 
@@ -276,6 +281,49 @@ A row carries the prints and the ids that stand for them: the address, the compe
 and age, the place, the name and school as printed with the dancer and school ids. The majors are
 read from `majors.sqlite`, fetched on first use.
 
+## Every row, streamed
+
+```ts
+for await (const row of rel.results.stream({ scope: "majors" })) { ... }
+for await (const row of rel.results.stream({
+  scope: "platform",            // "majors" | "transcribed" | "platform" | "all"
+  primary: true,                // only rows no other source stands in for
+  year: [2015, 2024],           // a year or an inclusive range
+  feis_level: [600, 700],       // also: event (one or several), tier, category, dancer_id, school_id
+})) { ... }
+for await (const entry of rel.results.entries({ scope: "all", event: "<event>" })) { ... }
+```
+
+A stream yields one row at a time and never holds the table in memory. A row carries its
+competition beside its own fields: `competition`, `event`, `year`, `tier`, `comp_no`, `title`,
+`category`, `age`, `age_printed`, `gender`, `competitors`, `feis_level`, `level_printed`,
+`date_printed`, `source`, `round`, and at a feis its `feis_id`, `feis_date_lo` and
+`feis_date_hi`. Its own: `address`, `section`, `card`, `place`, `place_printed`, `tie`,
+`relative_place`, `atts`, `withdrawn`, `qualifier`, `name_printed`, `name`, `school_printed`,
+`school_loc`, `school_id`, `dancer_id`, `dancer_scope`, `yob`, `yob_printed`, `points_printed`,
+`platform_dancer_id`, `primary_source`. Rows come in the release's own order, a competition's
+rows together.
+
+The scopes: `majors` reads the majors (tiers T1 to T3), `transcribed` the transcribed rows of
+every tier, `platform` every row a results platform published, `all` every row.
+
+AN ENTRY is one row per address, the row the card stands for: a card printed in several sections
+(a recall order, the final, a single round's solo placing, a qualifying heat) keeps the final's
+row, then the sectionless page's, the recall order's, the standings', the unplaced row, a heat's
+and last a solo round's (`entryRank`), and two rows of one rank keep the later. `recall_place` is
+the place the address's recall row carries.
+
+## Bulk reads
+
+```ts
+await rel.dancers.records();                   // every dancer: display, appearances, birth_lo, birth_hi,
+                                               // birth_source, birth_conflict, genders, scope
+await rel.dancers.records({ ids: [1001] });
+await rel.dancers.years({ ids: [1001], year: 2025 });  // region, region_source, nationals, geohash
+await rel.feises.dates();                      // every feis id with its first and last day
+await rel.majors.tabulated({ min_true_round: 3 });     // the majors addresses marked to round 3 or later
+```
+
 ## Teams and event-years
 
 ```ts
@@ -426,6 +474,13 @@ and the competitions still resolve.
                [--population girls|boys] [--rows]
     idq results [--release <spec>] [--dancer <id>] [--feis <id>] [--comp <c>] [--school <s>] [--year <y>] [--limit <n>]
     idq majors [--release <spec>] [--event <e>] [--year <y>] [--comp <c>] [--dancer <id>] [--rows]
+    idq stream [--release <spec>] --scope majors|transcribed|platform|all [--entries] [--event <e,...>]
+               [--year <y> | --year <from>..<to>] [--tier <t,...>] [--category <c,...>]
+               [--level <n,...>] [--dancer <id,...>] [--school <school_id>] [--primary]
+    idq records [--release <spec>] [--dancer <id,...>]
+    idq records --years [--release <spec>] [--dancer <id,...>] [--year <y>]
+    idq feis-dates [--release <spec>]
+    idq tabulated [--release <spec>] [--min-round <n>]
     idq teams [--release <spec>] [--event <e>] [--year <y>] [--rosters | --joins]
     idq events [--release <spec>] [--event <e>] [--year <y>]
     idq teacher [--release <spec>] <name|TD number>
@@ -437,7 +492,7 @@ and the competitions still resolve.
     idq resolve dancer [<name>] [--school s] [--comp <feis>/<code> | --yob y | --band b --year y]
     idq resolve registrations
 
-`dancer` prints the results of the one dancer where the answer is unique (every candidate's with
+`stream` prints one JSON line per row (or entry), as it reads them. `dancer` prints the results of the one dancer where the answer is unique (every candidate's with
 `--rows`). `--json` prints the call's answer as it is. `--cache` and `--repo` override the defaults.
 
 `resolve` speaks JSON lines: one answer per line out, and with no positional question the

package/dist/chunks/{index-v8e0qe9f.js → index-8gna56w2.js}

@@ -106,8 +106,149 @@ function nameLevel2(q, c) {
   return null;
 }
 
+var ENTRY_RANK = new Map([
+  ["final", 0],
+  ["", 1],
+  ["recall", 2],
+  ["standings", 3],
+  ["unplaced", 4],
+  ["qual", 5],
+  ["qual-a", 5],
+  ["qual-b", 5]
+]);
+function entryRank2(section) {
+  const known = ENTRY_RANK.get(section);
+  if (known !== undefined)
+    return known;
+  if (/^solo(?:-r\d+)?$/.test(section))
+    return 6;
+  return 9;
+}
+var COLUMNS = `r.address AS address, r.competition AS competition, c.event AS event,
+	c.year AS year, c.tier AS tier, c.comp_no AS comp_no, c.title AS title, c.category AS category,
+	c.age AS age, c.age_printed AS age_printed, c.gender AS gender, c.competitors AS competitors,
+	c.feis_level AS feis_level, c.level_printed AS level_printed, c.date_printed AS date_printed,
+	c.source AS source, c.round AS round, r.section AS section, r.card AS card, r.place AS place,
+	r.place_printed AS place_printed, r.tie AS tie, r.relative_place AS relative_place,
+	r.atts AS atts, r.withdrawn AS withdrawn, r.qualifier AS qualifier,
+	r.name_printed AS name_printed, r.name AS name, r.school_printed AS school_printed,
+	r.school_loc AS school_loc, r.school_id AS school_id, r.dancer_id AS dancer_id, r.yob AS yob,
+	r.yob_printed AS yob_printed, r.points_printed AS points_printed,
+	r.platform_dancer_id AS platform_dancer_id, r.primary_source AS primary_source,
+	r.file AS file`;
+var TRANSCRIBED = "('transcript', 'manifested')";
+var scopeDb = (s) => s === "majors" ? "majors" : "results";
+function streamSql(q) {
+  const where = [];
+  const args = [];
+  const list = (col, v, nocase = false) => {
+    if (v.length === 0) {
+      where.push("0");
+      return;
+    }
+    where.push(`${col}${nocase ? " COLLATE NOCASE" : ""} IN (${v.map(() => "?").join(", ")})`);
+    args.push(...v);
+  };
+  if (q.scope === "transcribed")
+    where.push(`r.source IN ${TRANSCRIBED}`);
+  else if (q.scope === "platform")
+    where.push(`r.source NOT IN ${TRANSCRIBED}`);
+  else if (q.scope !== "majors" && q.scope !== "all")
+    throw new Error(`archive-client: no stream scope "${String(q.scope)}"`);
+  if (q.event !== undefined)
+    list("c.event", typeof q.event === "string" ? [q.event] : q.event, true);
+  if (typeof q.year === "number") {
+    where.push("c.year = ?");
+    args.push(q.year);
+  } else if (Array.isArray(q.year)) {
+    where.push("c.year BETWEEN ? AND ?");
+    args.push(q.year[0], q.year[1]);
+  }
+  if (q.tier !== undefined)
+    list("c.tier", q.tier);
+  if (q.category !== undefined)
+    list("c.category", q.category);
+  if (q.feis_level !== undefined)
+    list("c.feis_level", q.feis_level);
+  if (q.dancer_id !== undefined)
+    list("r.dancer_id", q.dancer_id);
+  if (q.school_id !== undefined) {
+    where.push("r.school_id = ?");
+    args.push(q.school_id);
+  }
+  if (q.primary === true)
+    where.push("(r.primary_source IS NULL OR r.primary_source = '')");
+  return {
+    sql: `SELECT ${COLUMNS} FROM results r JOIN competitions c ON c.file = r.file AND c.line = r.competition_line${where.length === 0 ? "" : ` WHERE ${where.join(" AND ")}`} ORDER BY r.file, r.line`,
+    args
+  };
+}
+function feisDates(db) {
+  const out = new Map;
+  for (const f of db.query("SELECT feis_id, min(date_lo) AS lo, max(date_hi) AS hi FROM feises GROUP BY feis_id").all())
+    out.set(f.feis_id, { lo: f.lo, hi: f.hi });
+  return out;
+}
+var text = (v) => v === null || v === "" ? null : v;
+function rowOf(r, dates) {
+  const { file: _file, atts, withdrawn, ...rest } = r;
+  const d = dates.get(r.event);
+  return {
+    ...rest,
+    feis_id: d === undefined ? null : r.event,
+    feis_date_lo: d?.lo ?? null,
+    feis_date_hi: d?.hi ?? null,
+    atts: atts === "" ? [] : atts.split(","),
+    withdrawn: text(withdrawn),
+    qualifier: text(rest.qualifier),
+    primary_source: text(rest.primary_source),
+    dancer_scope: r.dancer_id === null ? null : r.dancer_id >= FEIS_LOCAL_ID_FLOOR2 ? "per-release" : "stable"
+  };
+}
+async function* streamRows(db, q) {
+  const dates = feisDates(db);
+  const { sql, args } = streamSql(q);
+  const stmt = db.prepare(sql);
+  try {
+    for (const r of stmt.iterate(...args))
+      yield rowOf(r, dates);
+  } finally {
+    stmt.finalize();
+  }
+}
+async function* entryRows(db, q) {
+  const dates = feisDates(db);
+  const { sql, args } = streamSql(q);
+  const stmt = db.prepare(sql);
+  let file = null;
+  let held = new Map;
+  const flush = function* () {
+    for (const h of held.values())
+      yield { ...rowOf(h.raw, dates), recall_place: h.recall };
+    held = new Map;
+  };
+  try {
+    for (const r of stmt.iterate(...args)) {
+      if (r.file !== file) {
+        yield* flush();
+        file = r.file;
+      }
+      const rank = entryRank2(r.section);
+      const h = held.get(r.address);
+      const recall = r.section === "recall" ? r.place : h?.recall ?? null;
+      if (h === undefined || rank <= h.rank)
+        held.set(r.address, { raw: r, rank, recall });
+      else
+        h.recall = recall;
+    }
+    yield* flush();
+  } finally {
+    stmt.finalize();
+  }
+}
+
 var FEIS_LOCAL_ID_FLOOR2 = 17179869184;
-var COLUMNS = `r.address AS address, r.competition AS competition, c.event AS event, c.year AS year,
+var COLUMNS2 = `r.address AS address, r.competition AS competition, c.event AS event, c.year AS year,
 	c.tier AS tier, c.title AS title, c.age AS age, c.category AS category, c.gender AS gender,
 	c.feis_level AS feis_level, c.comp_no AS comp_no, r.place AS place,
 	r.place_printed AS place_printed, c.competitors AS competitors, r.name_printed AS name_printed,
@@ -139,7 +280,7 @@ function queryResults(rel, db, q) {
     args.push(q.year);
   }
   const limit = q.limit ?? 500;
-  const all = rel.db(db).query(`SELECT ${COLUMNS} FROM results r JOIN competitions c ON c.file = r.file AND c.line = r.competition_line
+  const all = rel.db(db).query(`SELECT ${COLUMNS2} FROM results r JOIN competitions c ON c.file = r.file AND c.line = r.competition_line
 			 WHERE ${where.join(" AND ")}
 			 ORDER BY c.year, c.event, r.competition, r.place IS NULL, r.place, r.address`).all(...args);
   return {
@@ -164,6 +305,18 @@ class ResultFinder2 {
     await this.rel.require("results.level");
     return queryResults(this.rel, "results", q);
   }
+  async streamDb(q) {
+    const db = scopeDb(q.scope);
+    await this.rel.fetch(db);
+    await this.rel.require(db === "majors" ? "majors.stream" : "results.stream");
+    return this.rel.db(db);
+  }
+  async* stream(q) {
+    yield* streamRows(await this.streamDb(q), q);
+  }
+  async* entries(q) {
+    yield* entryRows(await this.streamDb(q), q);
+  }
 }
 
 class MajorsFinder2 {
@@ -208,6 +361,13 @@ class MajorsFinder2 {
 					 ORDER BY c.year, c.seq, c.competition`).all(...args).map((c) => ({ ...c, rows: rows.get(c.competition) ?? 0 }));
     return { tag: this.rel.tag, competitions };
   }
+  async tabulated(q = {}) {
+    const db = await this.db();
+    await this.rel.require("majors.tabs");
+    const min = q.min_true_round ?? 3;
+    const addresses = db.query("SELECT DISTINCT address FROM tabs WHERE true_round >= ? ORDER BY address").all(min).map((r) => r.address);
+    return { tag: this.rel.tag, min_true_round: min, addresses };
+  }
   async results(q) {
     await this.db();
     await this.rel.require("results.level");
@@ -365,6 +525,12 @@ var SCHOOL_WEIGHT = {
   unresolved: 0,
   unasked: 0
 };
+function idChunks(ids) {
+  const out = [];
+  for (let i = 0;i < ids.length; i += 500)
+    out.push(ids.slice(i, i + 500));
+  return out;
+}
 
 class DancerResolver2 {
   rel;
@@ -372,6 +538,34 @@ class DancerResolver2 {
   constructor(rel) {
     this.rel = rel;
   }
+  async records(q = {}) {
+    await this.rel.fetch("results");
+    await this.rel.require("dancers.records");
+    const db = this.rel.db("results");
+    const sql = "SELECT dancer_id, display, appearances, birth_lo, birth_hi, birth_source, birth_conflict, genders FROM dancers";
+    const rows = q.ids === undefined ? db.query(`${sql} ORDER BY dancer_id`).all() : idChunks(q.ids).flatMap((c) => db.query(`${sql} WHERE dancer_id IN (${c.map(() => "?").join(",")})`).all(...c));
+    rows.sort((a, b) => a.dancer_id - b.dancer_id);
+    return {
+      tag: this.rel.tag,
+      records: rows.map((r) => ({
+        ...r,
+        scope: r.dancer_id >= FEIS_LOCAL_ID_FLOOR2 ? "per-release" : "stable",
+        birth_conflict: r.birth_conflict === 1,
+        genders: r.genders === "" ? [] : r.genders.split(",")
+      }))
+    };
+  }
+  async years(q = {}) {
+    await this.rel.fetch("results");
+    await this.rel.require("dancers.years");
+    const db = this.rel.db("results");
+    const sql = "SELECT dancer_id, year, region, region_source, nationals, nationals_source, geohash FROM dancer_years";
+    const yearSql = q.year === undefined ? "" : " AND year = ?";
+    const yearArgs = q.year === undefined ? [] : [q.year];
+    const rows = q.ids === undefined ? db.query(`${sql} WHERE 1${yearSql} ORDER BY dancer_id, year`).all(...yearArgs) : idChunks(q.ids).flatMap((c) => db.query(`${sql} WHERE dancer_id IN (${c.map(() => "?").join(",")})${yearSql}`).all(...c, ...yearArgs));
+    rows.sort((a, b) => a.dancer_id - b.dancer_id || a.year - b.year);
+    return { tag: this.rel.tag, years: rows };
+  }
   async resolve(q) {
     const db = await this.rel.indexDb("dancers");
     const asked = dancerName(q.name, q.order === "unknown" ? undefined : q.order);
@@ -1067,6 +1261,12 @@ class FeisFinder2 {
   constructor(rel) {
     this.rel = rel;
   }
+  async dates() {
+    await this.rel.fetch("results");
+    await this.rel.require("feises.dates");
+    const dates = this.rel.db("results").query("SELECT feis_id, min(date_lo) AS date_lo, max(date_hi) AS date_hi FROM feises GROUP BY feis_id ORDER BY feis_id").all();
+    return { tag: this.rel.tag, dates };
+  }
   async find(q) {
     const db = await this.rel.indexDb("feises");
     const where = [];
@@ -2298,6 +2498,74 @@ var FEATURES2 = [
     since: 265,
     calls: "comps.resolve, comps.syllabus, comps.syllabusFor, registrations.resolve (the comps index)"
   },
+  {
+    feature: "results.stream",
+    needs: [
+      "results:competitions.round",
+      "results:competitions.feis_level",
+      "results:competitions.level_printed",
+      "results:competitions.age_printed",
+      "results:results.yob_printed",
+      "results:results.primary_source",
+      "results:results.platform_dancer_id",
+      "results:results.relative_place",
+      "results:feises.date_lo",
+      "results:feises.date_hi"
+    ],
+    since: 255,
+    calls: "results.stream, results.entries (scopes transcribed, platform, all)"
+  },
+  {
+    feature: "majors.stream",
+    needs: [
+      "majors:competitions.round",
+      "majors:competitions.feis_level",
+      "majors:results.yob_printed",
+      "majors:results.primary_source",
+      "majors:results.relative_place",
+      "majors:feises.date_lo"
+    ],
+    since: 255,
+    calls: "results.stream, results.entries (scope majors)"
+  },
+  {
+    feature: "majors.tabs",
+    needs: ["majors:tabs.address", "majors:tabs.true_round"],
+    since: 215,
+    calls: "majors.tabulated"
+  },
+  {
+    feature: "feises.dates",
+    needs: ["results:feises.feis_id", "results:feises.date_lo", "results:feises.date_hi"],
+    since: 245,
+    calls: "feises.dates"
+  },
+  {
+    feature: "dancers.records",
+    needs: [
+      "results:dancers.display",
+      "results:dancers.appearances",
+      "results:dancers.birth_lo",
+      "results:dancers.birth_hi",
+      "results:dancers.birth_source",
+      "results:dancers.birth_conflict",
+      "results:dancers.genders"
+    ],
+    since: 215,
+    calls: "dancers.records"
+  },
+  {
+    feature: "dancers.years",
+    needs: [
+      "results:dancer_years.region",
+      "results:dancer_years.region_source",
+      "results:dancer_years.nationals",
+      "results:dancer_years.nationals_source",
+      "results:dancer_years.geohash"
+    ],
+    since: 245,
+    calls: "dancers.years"
+  },
   {
     feature: "teams",
     needs: [
@@ -2929,4 +3197,4 @@ function releases2(opts = {}) {
   return new ReleaseManager2(opts);
 }
 
-export { DATABASES2, planAsset2, sha256File2, unzstdTo2, NAME_LEVELS2, nameLevel2, FEIS_LOCAL_ID_FLOOR2, ResultFinder2, MajorsFinder2, nameOrderOf, DancerResolver2, readPopulation2, canonicalOf2, CompResolver2, NO_CALENDAR2, FeisFinder2, StaffFinder2, SchoolResolver2, TeamFinder2, EventFinder2, defaultBuilders2, indexPath2, indexKey2, isCurrent2, buildIndex2, ensureIndex2, listIndexes2, readPin2, youngest2, dancerResolverOf2, RegistrationResolver2, ghRemote2, httpsRemote2, fallbackRemote2, setRemote2, columnsOf2, FEATURES2, capabilityOf2, needsRelease2, resolveTag2, Release2, openRelease2, openCached2, ReleaseManager2, releases2 };
+export { DATABASES2, planAsset2, sha256File2, unzstdTo2, NAME_LEVELS2, nameLevel2, entryRank2, FEIS_LOCAL_ID_FLOOR2, ResultFinder2, MajorsFinder2, nameOrderOf, DancerResolver2, readPopulation2, canonicalOf2, CompResolver2, NO_CALENDAR2, FeisFinder2, StaffFinder2, SchoolResolver2, TeamFinder2, EventFinder2, defaultBuilders2, indexPath2, indexKey2, isCurrent2, buildIndex2, ensureIndex2, listIndexes2, readPin2, youngest2, dancerResolverOf2, RegistrationResolver2, ghRemote2, httpsRemote2, fallbackRemote2, setRemote2, columnsOf2, FEATURES2, capabilityOf2, needsRelease2, resolveTag2, Release2, openRelease2, openCached2, ReleaseManager2, releases2 };

package/dist/cli.js

@@ -4,7 +4,7 @@ import {
   nameOrderOf,
   openRelease2,
   releases2
-} from "./chunks/index-v8e0qe9f.js";
+} from "./chunks/index-8gna56w2.js";
 import"./chunks/index-hne1wadw.js";
 import"./chunks/index-hghdy980.js";
 
@@ -50,6 +50,21 @@ var USAGE = `usage: idq <verb> [--cache <dir>] [--repo <owner/name>] [--json]
   majors [--release <spec>] [--event <e>] [--year <y>] [--comp <competition>] [--dancer <id>]
                                     the majors: every event and its years; an event's
                                     competitions; a competition's, an event's or a dancer's rows
+  stream [--release <spec>] --scope majors|transcribed|platform|all [--entries] [--event <e,...>]
+         [--year <y> | --year <from>..<to>] [--tier <t,...>] [--category <c,...>]
+         [--level <feis_level,...>] [--dancer <id,...>] [--school <school_id>] [--primary]
+                                    every row a scope holds, one JSON line each, never the
+                                    whole table in memory; --entries one per address (the row
+                                    it stands for, with its recall order)
+  records [--release <spec>] [--dancer <id,...>]
+                                    every dancer's record: display, appearances, birth window,
+                                    populations
+  records --years [--release <spec>] [--dancer <id,...>] [--year <y>]
+                                    each dancer's region, nationals and place per year
+  feis-dates [--release <spec>]     every feis's first and last day
+  tabulated [--release <spec>] [--min-round <n>]
+                                    the majors addresses the marks sheets tabulate to round n
+                                    or later (default 3)
   teams [--release <spec>] [--event <e>] [--year <y>] [--rosters | --joins]
                                     the team result rows of an event-year; with --rosters its
                                     roster teams and their dancers; with --joins how each roster
@@ -107,7 +122,12 @@ var VALUE = new Set([
   "--limit",
   "--event",
   "--title",
-  "--for"
+  "--for",
+  "--scope",
+  "--tier",
+  "--category",
+  "--level",
+  "--min-round"
 ]);
 function parseArgs(argv) {
   const positional = [];
@@ -319,6 +339,45 @@ async function resolveVerb(rel, what, args, p, say, stdin) {
       throw new Error("resolve comp|dancer|registrations");
   }
 }
+var SCOPES = ["majors", "transcribed", "platform", "all"];
+var listOf = (v) => v.split(",").map((x) => x.trim()).filter((x) => x !== "");
+var idsOf = (v) => v === undefined ? undefined : listOf(v).map(Number);
+function streamQueryOf(p) {
+  const scope = p.flags.get("--scope");
+  if (scope === undefined || !SCOPES.includes(scope))
+    throw new Error(`stream takes --scope ${SCOPES.join("|")}`);
+  const q = { scope };
+  const event = p.flags.get("--event");
+  if (event !== undefined)
+    q.event = listOf(event);
+  const year = p.flags.get("--year");
+  if (year !== undefined) {
+    const m = /^(\d{4})\.\.(\d{4})$/.exec(year);
+    q.year = m === null ? Number(year) : [Number(m[1]), Number(m[2])];
+    if (typeof q.year === "number" && !Number.isInteger(q.year))
+      throw new Error(`--year takes a year or <from>..<to>, not "${year}"`);
+  }
+  for (const [flag, key] of [
+    ["--tier", "tier"],
+    ["--category", "category"]
+  ]) {
+    const v = p.flags.get(flag);
+    if (v !== undefined)
+      q[key] = listOf(v);
+  }
+  const level = p.flags.get("--level");
+  if (level !== undefined)
+    q.feis_level = listOf(level).map(Number);
+  const dancers = idsOf(p.flags.get("--dancer"));
+  if (dancers !== undefined)
+    q.dancer_id = dancers;
+  const school = p.flags.get("--school");
+  if (school !== undefined)
+    q.school_id = school;
+  if (p.bools.has("--primary"))
+    q.primary = true;
+  return q;
+}
 function eventYearOf(p) {
   const event = p.flags.get("--event");
   const year = p.flags.get("--year");
@@ -644,6 +703,79 @@ async function main(argv, io = {}) {
           rel.close();
         }
       }
+      case "stream": {
+        const q = streamQueryOf(p);
+        const rel = await openRelease2(spec, { ...opts, assets: [] });
+        try {
+          let n = 0;
+          const rows = p.bools.has("--entries") ? rel.results.entries(q) : rel.results.stream(q);
+          for await (const r of rows) {
+            say(JSON.stringify(r));
+            n++;
+          }
+          log(`${n} ${p.bools.has("--entries") ? "entries" : "rows"} in ${rel.tag}`);
+          return n === 0 ? 1 : 0;
+        } finally {
+          rel.close();
+        }
+      }
+      case "records": {
+        const rel = await openRelease2(spec, { ...opts, assets: [] });
+        try {
+          const ids = idsOf(p.flags.get("--dancer"));
+          if (p.bools.has("--years")) {
+            const year = p.flags.get("--year");
+            const a = await rel.dancers.years({
+              ...ids !== undefined ? { ids } : {},
+              ...year !== undefined ? { year: Number(year) } : {}
+            });
+            out(a, () => {
+              for (const y of a.years)
+                say(`  ${y.dancer_id}  ${y.year}  ${y.region ?? "-"} (${y.region_source ?? "-"})  nationals ${y.nationals ?? "-"}  ${y.geohash ?? "-"}`);
+              say(`${a.years.length} dancer-year(s) in ${a.tag}`);
+            });
+            return a.years.length === 0 ? 1 : 0;
+          }
+          const a = await rel.dancers.records(ids !== undefined ? { ids } : {});
+          out(a, () => {
+            for (const r of a.records)
+              say(`  ${r.dancer_id}  ${r.display}  ${r.appearances} entries  born ${r.birth_lo ?? "?"}..${r.birth_hi ?? "?"} (${r.birth_source}${r.birth_conflict ? ", conflict" : ""})  ${r.genders.join(",") || "-"}`);
+            say(`${a.records.length} dancer(s) in ${a.tag}`);
+          });
+          return a.records.length === 0 ? 1 : 0;
+        } finally {
+          rel.close();
+        }
+      }
+      case "feis-dates": {
+        const rel = await openRelease2(spec, { ...opts, assets: [] });
+        try {
+          const a = await rel.feises.dates();
+          out(a, () => {
+            for (const d of a.dates)
+              say(`  ${d.feis_id}  ${d.date_lo ?? "?"}..${d.date_hi ?? "?"}`);
+            say(`${a.dates.length} feis(es) in ${a.tag}`);
+          });
+          return a.dates.length === 0 ? 1 : 0;
+        } finally {
+          rel.close();
+        }
+      }
+      case "tabulated": {
+        const rel = await openRelease2(spec, { ...opts, assets: [] });
+        try {
+          const min = p.flags.get("--min-round");
+          const a = await rel.majors.tabulated(min !== undefined ? { min_true_round: Number(min) } : {});
+          out(a, () => {
+            for (const x of a.addresses)
+              say(`  ${x}`);
+            say(`${a.addresses.length} address(es) tabulated to round ${a.min_true_round} or later in ${a.tag}`);
+          });
+          return a.addresses.length === 0 ? 1 : 0;
+        } finally {
+          rel.close();
+        }
+      }
       case "teams": {
         const rel = await openRelease2(spec, { ...opts, assets: [] });
         try {
@@ -783,5 +915,6 @@ export {
   USAGE,
   jsonLines,
   main,
-  parseArgs
+  parseArgs,
+  streamQueryOf
 };

package/dist/index.js

@@ -6,6 +6,7 @@ import {
   unzstdTo2,
   NAME_LEVELS2,
   nameLevel2,
+  entryRank2,
   FEIS_LOCAL_ID_FLOOR2,
   ResultFinder2,
   MajorsFinder2,
@@ -44,7 +45,7 @@ import {
   openCached2,
   ReleaseManager2,
   releases2
-} from "./chunks/index-v8e0qe9f.js";
+} from "./chunks/index-8gna56w2.js";
 import {
   cacheRoot2,
   repoOf2
@@ -87,6 +88,7 @@ export {
   dancerResolverOf2 as dancerResolverOf,
   defaultBuilders2 as defaultBuilders,
   ensureIndex2 as ensureIndex,
+  entryRank2 as entryRank,
   fallbackRemote2 as fallbackRemote,
   ghRemote2 as ghRemote,
   httpsRemote2 as httpsRemote,

package/dist/types/cli.d.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env bun
 import type { OpenOptions } from "./release.js";
-export declare const USAGE = "usage: idq <verb> [--cache <dir>] [--repo <owner/name>] [--json]\n\n  config repo <owner/name>          set the user-local repository the releases come from\n  list [--remote]                   the cached releases (size, files, indexes, the pin), and\n                                    with --remote the published tags\n  fetch [--release <spec>] [--assets results,majors,<file>...]\n                                    download, verify and cache a release (default: results)\n  pin <spec>                        the user-local pin: what a bare open (spec \"pinned\") opens\n  unpin                             remove the pin (a bare open is then \"latest\")\n  current                           the pin, and whether it is cached\n  add <dir> --tag <tag>             add a release built locally (its MANIFEST.json and files)\n                                    to the cache under its tag\n  index [--release <spec>] [<name>...] [--force]\n                                    build the per-tag indexes (default: every one)\n  verify [--release <spec>]         re-hash every cached file against its receipt\n  capabilities [--release <spec>]   what the release carries of what the client reads, from\n                                    its own schema: each feature full or missing (its calls\n                                    refuse, naming the first cut that publishes it)\n  prune [--keep N] [--dry-run]      remove all but the N newest cached releases (default 1)\n                                    and the pinned one\n  rm <tag> [--dry-run]              remove one cached release (not the pinned one)\n  drop-indexes [<tag>] [--dry-run]  remove the index files (of one release, or of every one)\n  feis [--release <spec>] [--platform <p>] [--code <c>] [--id <id>] [--name <glob>] [--year <y>]\n                                    the feises a query names\n  series [--release <spec>] <name|host code|feis id>\n                                    the series a name, a host code (fweb:ABC) or any feis id\n                                    names: its years, hosts, codes and next instance\n  school [--release <spec>] <print|school_id> [--region <code>]\n                                    a school print or id to the published schools\n  dancer [--release <spec>] <name> | --surname <s> | --like <glob> | --id <dancer_id>\n         [--order surname-first|given-first] [--school <print|id>]\n         [--yob <y> | --band <U10> [--year <y>] | --comp <competition>] [--as-of <date>]\n         [--population girls|boys] [--rows]\n                                    the dancers a name reaches, each with its evidence\n                                    (name level, age, school, population) and the arm; the results of\n                                    the one dancer where one answers (or --rows for each)\n  results [--release <spec>] [--dancer <id>] [--feis <id>] [--comp <competition>]\n          [--school <print|id>] [--year <y>] [--limit <n>]\n                                    the published result rows, prints and ids\n  majors [--release <spec>] [--event <e>] [--year <y>] [--comp <competition>] [--dancer <id>]\n                                    the majors: every event and its years; an event's\n                                    competitions; a competition's, an event's or a dancer's rows\n  teams [--release <spec>] [--event <e>] [--year <y>] [--rosters | --joins]\n                                    the team result rows of an event-year; with --rosters its\n                                    roster teams and their dancers; with --joins how each roster\n                                    team meets the results\n  events [--release <spec>] [--event <e>] [--year <y>]\n                                    each event-year's dates and place\n  teacher [--release <spec>] <name|TD number>\n                                    the staff lists' people: where they taught, the register,\n                                    where they sat\n  adjudicator [--release <spec>] <name|TD number>\n                                    the register's and the panels' people: the same answer\n  comp [--release <spec>] <feis> <code> [--year <y>] [--title <t>]\n                                    a feis's own comp code to the comp it names, through that\n                                    feis's syllabus (or the prior year's)\n  syllabus [--release <spec>] <feis> [year]\n                                    one feis-year's comps, in printed order\n  syllabus [--release <spec>] --for <series> [year]\n                                    a series' syllabus for a year: that year's where held,\n                                    else the newest, saying which and what it is read from\n  resolve comp [<feis> <code>] [--year <y>] [--title <t>]\n  resolve dancer [<name>] [--school <s>] [--comp <feis>/<code> | --yob <y> | --band <b> --year <y>]\n                 [--order <o>] [--as-of <date>] [--population girls|boys]\n  resolve registrations\n                                    JSONL out, one answer per line; with no positional the\n                                    questions are JSONL on stdin, one per line (a comp:\n                                    {feis, code, year?, title?}, a dancer: {name, school?, age?,\n                                    population?}, a registration: {name, school?, codes[], feis,\n                                    year?, population?})\n\n  <spec>: pinned (the default) | latest | a cut (42, 42.1) | a tag (archive-YYYY.MM.DD-N)";
+import type { StreamQuery } from "./stream.js";
+export declare const USAGE = "usage: idq <verb> [--cache <dir>] [--repo <owner/name>] [--json]\n\n  config repo <owner/name>          set the user-local repository the releases come from\n  list [--remote]                   the cached releases (size, files, indexes, the pin), and\n                                    with --remote the published tags\n  fetch [--release <spec>] [--assets results,majors,<file>...]\n                                    download, verify and cache a release (default: results)\n  pin <spec>                        the user-local pin: what a bare open (spec \"pinned\") opens\n  unpin                             remove the pin (a bare open is then \"latest\")\n  current                           the pin, and whether it is cached\n  add <dir> --tag <tag>             add a release built locally (its MANIFEST.json and files)\n                                    to the cache under its tag\n  index [--release <spec>] [<name>...] [--force]\n                                    build the per-tag indexes (default: every one)\n  verify [--release <spec>]         re-hash every cached file against its receipt\n  capabilities [--release <spec>]   what the release carries of what the client reads, from\n                                    its own schema: each feature full or missing (its calls\n                                    refuse, naming the first cut that publishes it)\n  prune [--keep N] [--dry-run]      remove all but the N newest cached releases (default 1)\n                                    and the pinned one\n  rm <tag> [--dry-run]              remove one cached release (not the pinned one)\n  drop-indexes [<tag>] [--dry-run]  remove the index files (of one release, or of every one)\n  feis [--release <spec>] [--platform <p>] [--code <c>] [--id <id>] [--name <glob>] [--year <y>]\n                                    the feises a query names\n  series [--release <spec>] <name|host code|feis id>\n                                    the series a name, a host code (fweb:ABC) or any feis id\n                                    names: its years, hosts, codes and next instance\n  school [--release <spec>] <print|school_id> [--region <code>]\n                                    a school print or id to the published schools\n  dancer [--release <spec>] <name> | --surname <s> | --like <glob> | --id <dancer_id>\n         [--order surname-first|given-first] [--school <print|id>]\n         [--yob <y> | --band <U10> [--year <y>] | --comp <competition>] [--as-of <date>]\n         [--population girls|boys] [--rows]\n                                    the dancers a name reaches, each with its evidence\n                                    (name level, age, school, population) and the arm; the results of\n                                    the one dancer where one answers (or --rows for each)\n  results [--release <spec>] [--dancer <id>] [--feis <id>] [--comp <competition>]\n          [--school <print|id>] [--year <y>] [--limit <n>]\n                                    the published result rows, prints and ids\n  majors [--release <spec>] [--event <e>] [--year <y>] [--comp <competition>] [--dancer <id>]\n                                    the majors: every event and its years; an event's\n                                    competitions; a competition's, an event's or a dancer's rows\n  stream [--release <spec>] --scope majors|transcribed|platform|all [--entries] [--event <e,...>]\n         [--year <y> | --year <from>..<to>] [--tier <t,...>] [--category <c,...>]\n         [--level <feis_level,...>] [--dancer <id,...>] [--school <school_id>] [--primary]\n                                    every row a scope holds, one JSON line each, never the\n                                    whole table in memory; --entries one per address (the row\n                                    it stands for, with its recall order)\n  records [--release <spec>] [--dancer <id,...>]\n                                    every dancer's record: display, appearances, birth window,\n                                    populations\n  records --years [--release <spec>] [--dancer <id,...>] [--year <y>]\n                                    each dancer's region, nationals and place per year\n  feis-dates [--release <spec>]     every feis's first and last day\n  tabulated [--release <spec>] [--min-round <n>]\n                                    the majors addresses the marks sheets tabulate to round n\n                                    or later (default 3)\n  teams [--release <spec>] [--event <e>] [--year <y>] [--rosters | --joins]\n                                    the team result rows of an event-year; with --rosters its\n                                    roster teams and their dancers; with --joins how each roster\n                                    team meets the results\n  events [--release <spec>] [--event <e>] [--year <y>]\n                                    each event-year's dates and place\n  teacher [--release <spec>] <name|TD number>\n                                    the staff lists' people: where they taught, the register,\n                                    where they sat\n  adjudicator [--release <spec>] <name|TD number>\n                                    the register's and the panels' people: the same answer\n  comp [--release <spec>] <feis> <code> [--year <y>] [--title <t>]\n                                    a feis's own comp code to the comp it names, through that\n                                    feis's syllabus (or the prior year's)\n  syllabus [--release <spec>] <feis> [year]\n                                    one feis-year's comps, in printed order\n  syllabus [--release <spec>] --for <series> [year]\n                                    a series' syllabus for a year: that year's where held,\n                                    else the newest, saying which and what it is read from\n  resolve comp [<feis> <code>] [--year <y>] [--title <t>]\n  resolve dancer [<name>] [--school <s>] [--comp <feis>/<code> | --yob <y> | --band <b> --year <y>]\n                 [--order <o>] [--as-of <date>] [--population girls|boys]\n  resolve registrations\n                                    JSONL out, one answer per line; with no positional the\n                                    questions are JSONL on stdin, one per line (a comp:\n                                    {feis, code, year?, title?}, a dancer: {name, school?, age?,\n                                    population?}, a registration: {name, school?, codes[], feis,\n                                    year?, population?})\n\n  <spec>: pinned (the default) | latest | a cut (42, 42.1) | a tag (archive-YYYY.MM.DD-N)";
 export interface Parsed {
     positional: string[];
     flags: Map<string, string>;
@@ -15,6 +16,7 @@ type Line<T> = {
     error: string;
 };
 export declare function jsonLines<T>(text: string, check: (v: T) => string | null): Line<T>[];
+export declare function streamQueryOf(p: Parsed): StreamQuery;
 export declare function main(argv: readonly string[], io?: {
     say?: (s: string) => void;
     log?: (s: string) => void;

package/dist/types/dancers.d.ts

@@ -140,10 +140,43 @@ export interface DancerRecord {
 export declare const PACKAGE_KEY_PARTS: readonly string[];
 export declare const dancersIndex: IndexBuilder;
 export declare function yearOf(asOf: string | undefined): number | null;
+export interface DancerBulkRecord {
+    dancer_id: number;
+    display: string;
+    scope: "per-release" | "stable";
+    appearances: number;
+    birth_lo: number | null;
+    birth_hi: number | null;
+    birth_source: string;
+    birth_conflict: boolean;
+    genders: string[];
+}
+export interface DancerYear {
+    dancer_id: number;
+    year: number;
+    region: string | null;
+    region_source: string | null;
+    nationals: string | null;
+    nationals_source: string | null;
+    geohash: string | null;
+}
 export declare class DancerResolver {
     private readonly rel;
     private readonly printIds;
     constructor(rel: Release);
+    records(q?: {
+        ids?: readonly number[];
+    }): Promise<{
+        tag: string;
+        records: DancerBulkRecord[];
+    }>;
+    years(q?: {
+        ids?: readonly number[];
+        year?: number;
+    }): Promise<{
+        tag: string;
+        years: DancerYear[];
+    }>;
     resolve(q: DancerQuery): Promise<DancerAnswer>;
     search(q: DancerSearchQuery): Promise<DancerAnswer>;
     private capped;

package/dist/types/feises.d.ts

@@ -77,6 +77,14 @@ export declare const NO_CALENDAR = "the release holds a feis-year once its resul
 export declare class FeisFinder {
     private readonly rel;
     constructor(rel: Release);
+    dates(): Promise<{
+        tag: string;
+        dates: {
+            feis_id: string;
+            date_lo: string | null;
+            date_hi: string | null;
+        }[];
+    }>;
     find(q: FeisQuery): Promise<FeisAnswer>;
     codeStem(token: string, stem: string): Promise<string[]>;
     seriesOf(brand: string): Promise<Series>;

package/dist/types/index.d.ts

@@ -1,6 +1,6 @@
 export { type AssetPlan, DATABASES, type DatabaseName, type LandedFile, type Receipt, type ReleaseManifest, planAsset, } from "./assets.js";
 export { sha256File, unzstdTo } from "./bytes.js";
-export { type AgeAsked, type AgeEvidence, type DancerAnswer, type DancerCandidate, type DancerQuery, type DancerRecord, DancerResolver, type DancerSchool, type DancerSearchQuery, type ExcludedCandidate, FEIS_LOCAL_ID_FLOOR, type PopulationEvidence, type SchoolEvidence, readPopulation, } from "./dancers.js";
+export { type AgeAsked, type AgeEvidence, type DancerAnswer, type DancerBulkRecord, type DancerCandidate, type DancerQuery, type DancerRecord, DancerResolver, type DancerSchool, type DancerSearchQuery, type DancerYear, type ExcludedCandidate, FEIS_LOCAL_ID_FLOOR, type PopulationEvidence, type SchoolEvidence, readPopulation, } from "./dancers.js";
 export { NAME_LEVELS, type NameLevel, nameLevel } from "./dancers-names.js";
 export { type Comp, type CompAnswer, type CompKind, type CompQuery, CompResolver, type CompSource, type FeisPlace, type SyllabusAnswer, type SyllabusForAnswer, type SyllabusSource, type TitleCheck, canonicalOf, } from "./comps.js";
 export { type ClientOptions, cacheRoot, repoOf } from "./config.js";
@@ -15,5 +15,6 @@ export { type MajorCompetition, type MajorEvent, MajorsFinder, type ResultAnswer
 export { type DancerAnswerLike, type DancerQueryLike, type DancerResolverLike, type DancerUnavailable, type RegistrationRow, type ResolvedDancer, RegistrationResolver, type ResolvedRegistration, dancerResolverOf, youngest, } from "./registrations.js";
 export { type SchoolAnswer, type SchoolCandidate, type SchoolQuery, SchoolResolver, type SchoolRung, } from "./schools.js";
 export { type Capability, FEATURES, type Feature, type SchemaProbe, capabilityOf, columnsOf, needsRelease, } from "./schema.js";
+export { type EntryRow, type StreamQuery, type StreamRow, type StreamScope, entryRank, } from "./stream.js";
 export { EventFinder, type EventYear, type EventYearQuery, type ResultRoster, type RosterMember, type RosterTeam, TeamFinder, type TeamResult, } from "./teams.js";
 export { ARCHIVE_TAG, type ArchiveTag, type ReleaseSpec, compareTagsNewestFirst, cutOf, parseArchiveTag, parseReleaseSpec, pickTag, renderArchiveTag, } from "./tag.js";

package/dist/types/results.d.ts

@@ -1,4 +1,5 @@
 import type { Release } from "./release.js";
+import { type EntryRow, type StreamQuery, type StreamRow } from "./stream.js";
 export declare const FEIS_LOCAL_ID_FLOOR = 17179869184;
 export interface ResultQuery {
     dancer_id?: number;
@@ -43,6 +44,9 @@ export declare class ResultFinder {
     private readonly rel;
     constructor(rel: Release);
     query(q: ResultQuery): Promise<ResultAnswer>;
+    private streamDb;
+    stream(q: StreamQuery): AsyncGenerator<StreamRow>;
+    entries(q: StreamQuery): AsyncGenerator<EntryRow>;
 }
 export interface MajorEvent {
     event: string;
@@ -78,5 +82,12 @@ export declare class MajorsFinder {
         tag: string;
         competitions: MajorCompetition[];
     }>;
+    tabulated(q?: {
+        min_true_round?: number;
+    }): Promise<{
+        tag: string;
+        min_true_round: number;
+        addresses: string[];
+    }>;
     results(q: ResultQuery): Promise<ResultAnswer>;
 }

package/dist/types/stream.d.ts

@@ -0,0 +1,71 @@
+import type { Database } from "bun:sqlite";
+export type StreamScope = "majors" | "transcribed" | "platform" | "all";
+export interface StreamQuery {
+    scope: StreamScope;
+    event?: string | string[];
+    year?: number | [number, number];
+    tier?: string[];
+    category?: string[];
+    feis_level?: number[];
+    dancer_id?: number[];
+    school_id?: string;
+    primary?: boolean;
+}
+export interface StreamRow {
+    address: string;
+    competition: string;
+    event: string;
+    year: number | null;
+    tier: string;
+    comp_no: string | null;
+    title: string | null;
+    category: string | null;
+    age: string | null;
+    age_printed: string | null;
+    gender: string | null;
+    competitors: number | null;
+    feis_level: number | null;
+    level_printed: string | null;
+    date_printed: string | null;
+    source: string;
+    round: string | null;
+    feis_id: string | null;
+    feis_date_lo: string | null;
+    feis_date_hi: string | null;
+    section: string;
+    card: string;
+    place: number | null;
+    place_printed: string;
+    tie: number;
+    relative_place: number | null;
+    atts: string[];
+    withdrawn: string | null;
+    qualifier: string | null;
+    name_printed: string;
+    name: string | null;
+    school_printed: string;
+    school_loc: string;
+    school_id: string | null;
+    dancer_id: number | null;
+    dancer_scope: "per-release" | "stable" | null;
+    yob: number | null;
+    yob_printed: string | null;
+    points_printed: string | null;
+    platform_dancer_id: string | null;
+    primary_source: string | null;
+}
+export interface EntryRow extends StreamRow {
+    recall_place: number | null;
+}
+export declare function entryRank(section: string): number;
+export declare const scopeDb: (s: StreamScope) => "majors" | "results";
+export declare function streamSql(q: StreamQuery): {
+    sql: string;
+    args: (string | number)[];
+};
+export declare function feisDates(db: Database): Map<string, {
+    lo: string | null;
+    hi: string | null;
+}>;
+export declare function streamRows(db: Database, q: StreamQuery): AsyncGenerator<StreamRow>;
+export declare function entryRows(db: Database, q: StreamQuery): AsyncGenerator<EntryRow>;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@finbheara/archive-client",
-	"version": "0.4.0",
+	"version": "0.5.0",
 	"description": "Open a versioned release of an Irish dance results archive: download, verify, decompress and cache it by tag, keep the cache, and resolve feises and schools against it",
 	"keywords": ["irish-dance", "results", "release", "cache", "sqlite"],
 	"license": "MIT",