@finbheara/archive-client 0.40.0 → 0.42.0

package/dist/cli.js

@@ -20,13 +20,15 @@ import {
   LATEST_CHECK_BUDGET_MS2,
   latestCheck2,
   within,
-  staleWarning2
-} from "./chunks/index-bs939h4v.js";
+  staleWarning2,
+  progressMode2,
+  progressRenderer2
+} from "./chunks/index-0xdcttqf.js";
 import"./chunks/index-hghdy980.js";
 
-var USAGE_HEADER = "usage: idq <verb> [--cache <dir>] [--repo <owner/name>] [--json] [--live]";
+var USAGE_HEADER = "usage: idq <verb> [--cache <dir>] [--repo <owner/name>] [--json] [--live] [--progress tty|plain|none]";
 var GLOBAL_FLAGS = {
-  value: ["--cache", "--repo"],
+  value: ["--cache", "--repo", "--progress"],
   bool: ["--json", "--live"]
 };
 var VERBS = [
@@ -510,6 +512,12 @@ var VERBS = [
   }
 ];
 var USAGE_FOOTER = [
+  "  A download or an index build reports on stderr: on a terminal one line per file, redrawn in",
+  "  place (bytes of the total, rate, downloading, unpacking, verifying, indexing, done); redirected,",
+  "  one start line and one done (or failed) line per file, with its size and time. --progress",
+  "  tty|plain|none (or IDQ_PROGRESS) forces a mode. Up to 4 files download at once, each",
+  "  decompressed and hashed while it arrives; the first failure cancels the rest, and fails the call.",
+  "",
   "  A query on the default spec prints one warning line on stderr when its release is older than",
   "  the newest published one (the remote asked at most once per hour, <cache>/LATEST.json); the",
   "  answer and the exit code are unchanged, and an offline check says nothing.",
@@ -897,8 +905,10 @@ async function main(argv, io = {}) {
   const say = io.say ?? ((s) => console.log(s));
   const log = io.log ?? ((s) => console.error(s));
   let p;
+  let progress;
   try {
     p = parseArgs(argv);
+    progress = progressRenderer2(progressMode2(p.flags.get("--progress"), io.env ?? process.env, io.isTTY ?? process.stderr.isTTY === true), { line: log, ...io.write !== undefined ? { write: io.write } : {} });
   } catch (e) {
     log(e.message);
     return 2;
@@ -909,6 +919,7 @@ async function main(argv, io = {}) {
   const opts = {
     ...io.opts,
     log,
+    progress: io.opts?.progress ?? progress.sink,
     ...p.flags.has("--cache") ? { cache: p.flags.get("--cache") } : {},
     ...p.flags.has("--repo") ? { repo: p.flags.get("--repo") } : {}
   };
@@ -1777,9 +1788,11 @@ async function main(argv, io = {}) {
         return verb === undefined || verb === "help" ? 0 : 2;
     }
   } catch (e) {
+    progress.close();
     log(`idq ${verb}: ${e.message}`);
     return 1;
   } finally {
+    progress.close();
     const b = answered.build;
     if (b !== null)
       log(`answered from ${b.build.label} (unpublished, ${b.build.dir}). Pass --release <tag> for a published one, or --live for the pin.`);

package/dist/index.js

@@ -4,6 +4,7 @@ import {
   planAsset2,
   sha256File2,
   unzstdTo2,
+  streamTo2,
   NAME_LEVELS2,
   nameLevel2,
   TabFinder2,
@@ -59,6 +60,7 @@ import {
   DEPARTURE_REASONS2,
   UpcomingFinder2,
   resolveTag2,
+  DEFAULT_CONCURRENCY2,
   Release2,
   openRelease2,
   openCached2,
@@ -79,8 +81,11 @@ import {
   latestCheckPath2,
   readLatestCheck2,
   latestCheck2,
-  staleWarning2
-} from "./chunks/index-bs939h4v.js";
+  staleWarning2,
+  PROGRESS_MODES2,
+  progressMode2,
+  progressRenderer2
+} from "./chunks/index-0xdcttqf.js";
 import {
   cacheRoot2,
   repoOf2
@@ -94,15 +99,86 @@ import {
   parseReleaseSpec2,
   pickTag2
 } from "./chunks/index-hghdy980.js";
+var ATTACH_OUTCOMES = [
+  "majors",
+  "no-card",
+  "attached",
+  "unrefuted",
+  "decision",
+  "account",
+  "unique-name",
+  "band-unique",
+  "pre-majors",
+  "transfer",
+  "address-conflict",
+  "no-name",
+  "no-year",
+  "no-namesake",
+  "school-unresolved",
+  "school-disagrees",
+  "school-years",
+  "sex-split",
+  "birth-window",
+  "under-cap",
+  "too-young",
+  "young-school",
+  "level-age",
+  "ambiguous",
+  "vetoed",
+  "feis-birth",
+  "co-attendance",
+  "card-split",
+  "same-spans-identities",
+  "same-contradicted",
+  "co-appearance",
+  "under-18-return",
+  "distinct-split"
+];
+var OUTCOME_UNREAD = 63;
+var OUTCOME_MASK = 63;
+var ATTACH_VIA = {
+  assignment: 1 << 6,
+  ledger: 1 << 7,
+  "local-attach": 1 << 8,
+  "local-fold": 1 << 9,
+  "feis-attach": 1 << 10,
+  kinship: 1 << 11,
+  "initial-only": 1 << 12,
+  "name-school-fold": 1 << 13,
+  "feis-local": 1 << 14,
+  ruling: 1 << 15
+};
+var CODE = new Map(ATTACH_OUTCOMES.map((n, i) => [n, i]));
+function outcomeCode(name) {
+  const c = CODE.get(name);
+  if (c === undefined)
+    throw new Error(`attach basis: no code for the outcome ${name}`);
+  return c;
+}
+function packAttachBasis(outcome, via) {
+  let n = outcome;
+  for (const v of via)
+    n |= ATTACH_VIA[v];
+  return n;
+}
+function unpackAttachBasis(n) {
+  const code = n & OUTCOME_MASK;
+  const outcome = code === OUTCOME_UNREAD ? "unread" : ATTACH_OUTCOMES[code] ?? null;
+  const via = Object.keys(ATTACH_VIA).filter((k) => (n & ATTACH_VIA[k]) !== 0);
+  return { outcome, via };
+}
 export {
   ADVANCEMENT_DANCES2 as ADVANCEMENT_DANCES,
   ARCHIVE_TAG2 as ARCHIVE_TAG,
+  ATTACH_OUTCOMES,
+  ATTACH_VIA,
   CALENDAR_STAGES2 as CALENDAR_STAGES,
   CALENDAR_STAGE_STATES2 as CALENDAR_STAGE_STATES,
   CALENDAR_TERMINALS2 as CALENDAR_TERMINALS,
   CalendarFinder2 as CalendarFinder,
   CompResolver2 as CompResolver,
   DATABASES2 as DATABASES,
+  DEFAULT_CONCURRENCY2 as DEFAULT_CONCURRENCY,
   DEFAULT_KEEP2 as DEFAULT_KEEP,
   DEPARTURE_REASONS2 as DEPARTURE_REASONS,
   DancerResolver2 as DancerResolver,
@@ -120,6 +196,9 @@ export {
   NAME_LEVELS2 as NAME_LEVELS,
   NO_CALENDAR2 as NO_CALENDAR,
   NO_UPCOMING2 as NO_UPCOMING,
+  OUTCOME_MASK,
+  OUTCOME_UNREAD,
+  PROGRESS_MODES2 as PROGRESS_MODES,
   PanelFinder2 as PanelFinder,
   RegistrationResolver2 as RegistrationResolver,
   Release2 as Release,
@@ -161,11 +240,15 @@ export {
   openBuild2 as openBuild,
   openCached2 as openCached,
   openRelease2 as openRelease,
+  outcomeCode,
+  packAttachBasis,
   parseArchiveTag2 as parseArchiveTag,
   parseReleaseSpec2 as parseReleaseSpec,
   pickBuild2 as pickBuild,
   pickTag2 as pickTag,
   planAsset2 as planAsset,
+  progressMode2 as progressMode,
+  progressRenderer2 as progressRenderer,
   readLatestCheck2 as readLatestCheck,
   readPin2 as readPin,
   readPopulation2 as readPopulation,
@@ -180,8 +263,10 @@ export {
   setRemote2 as setRemote,
   sha256File2 as sha256File,
   staleWarning2 as staleWarning,
+  streamTo2 as streamTo,
   sweepBuilds2 as sweepBuilds,
   syllabusSourceOf2 as syllabusSourceOf,
+  unpackAttachBasis,
   unzstdTo2 as unzstdTo,
   youngest2 as youngest
 };

package/dist/types/attach-basis.d.ts

@@ -0,0 +1,23 @@
+export declare const ATTACH_OUTCOMES: readonly ["majors", "no-card", "attached", "unrefuted", "decision", "account", "unique-name", "band-unique", "pre-majors", "transfer", "address-conflict", "no-name", "no-year", "no-namesake", "school-unresolved", "school-disagrees", "school-years", "sex-split", "birth-window", "under-cap", "too-young", "young-school", "level-age", "ambiguous", "vetoed", "feis-birth", "co-attendance", "card-split", "same-spans-identities", "same-contradicted", "co-appearance", "under-18-return", "distinct-split"];
+export declare const OUTCOME_UNREAD = 63;
+export declare const OUTCOME_MASK = 63;
+export type AttachOutcomeName = (typeof ATTACH_OUTCOMES)[number] | "unread";
+export declare const ATTACH_VIA: {
+    readonly assignment: number;
+    readonly ledger: number;
+    readonly "local-attach": number;
+    readonly "local-fold": number;
+    readonly "feis-attach": number;
+    readonly kinship: number;
+    readonly "initial-only": number;
+    readonly "name-school-fold": number;
+    readonly "feis-local": number;
+    readonly ruling: number;
+};
+export type AttachVia = keyof typeof ATTACH_VIA;
+export declare function outcomeCode(name: string): number;
+export declare function packAttachBasis(outcome: number, via: readonly AttachVia[]): number;
+export declare function unpackAttachBasis(n: number): {
+    outcome: AttachOutcomeName | null;
+    via: AttachVia[];
+};

package/dist/types/bytes.d.ts

@@ -1,3 +1,15 @@
 export declare function sha256File(path: string): Promise<string>;
 export declare function unzstdTo(src: string, dest: string): Promise<string>;
 export declare function sha256OfDigest(digest: string | null | undefined): string | null;
+export interface Streamed {
+    rawSha256: string;
+    rawBytes: number;
+    sha256: string;
+    bytes: number;
+}
+export declare function streamTo(source: AsyncIterable<Uint8Array>, dest: string, o: {
+    decompress: boolean;
+    raw?: string;
+    onBytes?: (received: number, written: number) => void;
+    signal?: AbortSignal;
+}): Promise<Streamed>;

package/dist/types/cli-spec.d.ts

@@ -4,7 +4,7 @@ export interface VerbSpec {
     bool: readonly string[];
     usage: readonly string[];
 }
-export declare const USAGE_HEADER = "usage: idq <verb> [--cache <dir>] [--repo <owner/name>] [--json] [--live]";
+export declare const USAGE_HEADER = "usage: idq <verb> [--cache <dir>] [--repo <owner/name>] [--json] [--live] [--progress tty|plain|none]";
 export declare const GLOBAL_FLAGS: {
     value: readonly string[];
     bool: readonly string[];

package/dist/types/cli.d.ts

@@ -39,4 +39,7 @@ export declare function main(argv: readonly string[], io?: {
     stdin?: () => Promise<string>;
     freshness?: FreshnessIO | false;
     local?: LocalIO | false;
+    env?: Readonly<Record<string, string | undefined>>;
+    isTTY?: boolean;
+    write?: (s: string) => void;
 }): Promise<number>;

package/dist/types/index.d.ts

@@ -1,5 +1,6 @@
 export { type AssetPlan, DATABASES, type DatabaseName, type LandedFile, type Receipt, type ReleaseManifest, planAsset, } from "./assets.js";
-export { sha256File, unzstdTo } from "./bytes.js";
+export { ATTACH_OUTCOMES, ATTACH_VIA, type AttachOutcomeName, type AttachVia, OUTCOME_MASK, OUTCOME_UNREAD, outcomeCode, packAttachBasis, unpackAttachBasis, } from "./attach-basis.js";
+export { type Streamed, sha256File, streamTo, unzstdTo } from "./bytes.js";
 export { type AgeAsked, type AgeEvidence, type DancerAnswer, type DancerBulkRecord, type DancerCandidate, type DancerQuery, type DancerRecord, DancerResolver, type DancerRegion, type DancerSchool, type DancerSearchQuery, type DancerYear, type ExcludedCandidate, FEIS_LOCAL_ID_FLOOR, type InitialsRefuted, type Narrowing, initialsOnly, type OireachtasAsked, type PopulationEvidence, type RegionEvidence, type SchoolEvidence, readPopulation, } from "./dancers.js";
 export { NAME_LEVELS, type NameLevel, nameLevel } from "./dancers-names.js";
 export { ADVANCEMENT_DANCES, type DancerGrades, type DancerLevelAnswer, type DancerLevelQuery, type LevelRange, } from "./dancer-level.js";
@@ -14,7 +15,8 @@ export { type Registered, type Sat, type StaffAnswer, StaffFinder, type StaffPer
 export { type FreshnessFloor, LATEST_CHECK_BUDGET_MS, LATEST_CHECK_MS, type LatestCheck, latestCheck, latestCheckPath, readLatestCheck, staleWarning, } from "./freshness.js";
 export { type Pin, readPin } from "./pin.js";
 export { type Remote, type RemoteAsset, fallbackRemote, ghRemote, httpsRemote, setRemote, } from "./remote.js";
-export { type OpenOptions, Release, openCached, openRelease, resolveTag, } from "./release.js";
+export { DEFAULT_CONCURRENCY, type Fetching, type OpenOptions, Release, openCached, openRelease, resolveTag, } from "./release.js";
+export { PROGRESS_MODES, type ProgressEvent, type ProgressMode, type ProgressRenderer, type ProgressSink, type ProgressState, progressMode, progressRenderer, } from "./progress.js";
 export { type MajorCompetition, type MajorEvent, MajorsFinder, type ResultAnswer, ResultFinder, type ResultQuery, type ResultRow, } from "./results.js";
 export { type DancerAnswerLike, type DancerQueryLike, type DancerResolverLike, type DancerUnavailable, type RegistrationRow, type ResolvedDancer, RegistrationResolver, type ResolvedRegistration, dancerResolverOf, youngest, } from "./registrations.js";
 export type { LineageEdge, ObservedAssociation, RegistrySchool, SchoolPrint, SchoolRedirect, } from "./school-bulk.js";

package/dist/types/indexes.d.ts

@@ -1,4 +1,5 @@
 import { Database } from "bun:sqlite";
+import type { ProgressSink } from "./progress.js";
 import type { Release } from "./release.js";
 export interface IndexBuilder {
     readonly name: string;
@@ -27,6 +28,7 @@ export declare function buildIndex(rel: Release, b: IndexBuilder, o: {
 export declare function ensureIndex(rel: Release, b: IndexBuilder, o: {
     force: boolean;
     log?: (line: string) => void;
+    progress?: ProgressSink;
 }): Promise<string>;
 export interface IndexFile {
     name: string;

package/dist/types/manager.d.ts

@@ -84,6 +84,7 @@ export declare class ReleaseManager {
             path: string;
         }[];
     }>;
+    private buildAsLanded;
     private plan;
     private planRemove;
     private apply;

package/dist/types/progress.d.ts

@@ -0,0 +1,30 @@
+export type ProgressState = "downloading" | "unpacking" | "verifying" | "indexing" | "done" | "failed" | "cancelled";
+export interface ProgressEvent {
+    tag: string;
+    kind: "asset" | "index";
+    item: string;
+    state: ProgressState;
+    bytes: number;
+    total: number | null;
+    unpacked: number;
+    ms: number;
+    error?: string;
+}
+export type ProgressSink = (e: ProgressEvent) => void;
+export type ProgressMode = "tty" | "plain" | "none";
+export declare const PROGRESS_MODES: readonly ProgressMode[];
+export declare function progressMode(flag: string | undefined, env: Readonly<Record<string, string | undefined>>, isTTY: boolean): ProgressMode;
+export declare function bytesText(n: number): string;
+export declare function plainLine(e: ProgressEvent, started: boolean): string | null;
+export declare function ttyLine(e: ProgressEvent, width: number): string;
+export interface ProgressRenderer {
+    readonly mode: ProgressMode;
+    sink: ProgressSink;
+    close(): void;
+}
+export declare function progressRenderer(mode: ProgressMode, io: {
+    line: (s: string) => void;
+    write?: (s: string) => void;
+    columns?: () => number;
+    intervalMs?: number;
+}): ProgressRenderer;

package/dist/types/release.d.ts

@@ -1,5 +1,6 @@
 import { Database } from "bun:sqlite";
 import { type AssetPlan, type LandedFile, type Receipt, type ReleaseManifest } from "./assets.js";
+import { type Streamed } from "./bytes.js";
 import { CalendarFinder } from "./calendar.js";
 import { CompResolver } from "./comps.js";
 import { type ClientOptions } from "./config.js";
@@ -7,8 +8,9 @@ import { DancerResolver } from "./dancers.js";
 import { FeisFinder } from "./feises.js";
 import { type IndexBuilder } from "./indexes.js";
 import { StaffFinder } from "./people.js";
+import type { ProgressSink } from "./progress.js";
 import { type DancerResolverLike, RegistrationResolver } from "./registrations.js";
-import { type Remote } from "./remote.js";
+import { type Remote, type RemoteAsset } from "./remote.js";
 import { MajorsFinder, ResultFinder } from "./results.js";
 import { type Capability } from "./schema.js";
 import { type SchoolReaderChoice, SchoolResolver } from "./schools.js";
@@ -22,6 +24,8 @@ export interface OpenOptions extends ClientOptions {
     strict?: boolean;
     keepCompressed?: boolean;
     log?: (line: string) => void;
+    progress?: ProgressSink;
+    concurrency?: number;
     builders?: readonly IndexBuilder[];
     schoolReader?: SchoolReaderChoice;
     dancerResolver?: DancerResolverLike | null;
@@ -30,6 +34,13 @@ export declare function tagDir(root: string, tag: string): string;
 export declare const receiptPath: (dir: string) => string;
 export declare function readReceipt(dir: string): Receipt | null;
 export declare function resolveTag(spec: string | number | ReleaseSpec, opts?: OpenOptions): Promise<string>;
+export declare const DEFAULT_CONCURRENCY = 4;
+export declare function partPath(dest: string): string;
+export interface Fetching {
+    each(asset: string): Promise<LandedFile>;
+    all: Promise<LandedFile[]>;
+    cancel(): void;
+}
 export declare class Release {
     readonly tag: string;
     readonly cut: number | undefined;
@@ -54,13 +65,17 @@ export declare class Release {
     readonly feisPanels: FeisPanelFinder;
     private readonly opts;
     private readonly handles;
-    private remoteAssets;
+    private listing;
     constructor(tag: string, root: string, manifest: ReleaseManifest, opts: OpenOptions);
     receipt(): Receipt;
     plan(asset: string): AssetPlan;
     has(asset: string): boolean;
     path(asset: string): string;
     fetch(...assets: string[]): Promise<LandedFile[]>;
+    private readonly inflight;
+    private slots;
+    private swept;
+    fetching(assets: readonly string[]): Fetching;
     publishes(asset: string): Promise<boolean>;
     private readonly probe;
     private readonly checked;
@@ -83,6 +98,23 @@ export declare class Release {
     close(): void;
     [Symbol.dispose](): void;
 }
+export declare function verifyLanded(got: Pick<Streamed, "rawSha256" | "sha256">, pub: {
+    name: string;
+    digest: string | null;
+}, plan: AssetPlan, strict: boolean): LandedFile["verified"];
+export interface AssetJob {
+    remote: Remote;
+    repo: string;
+    tag: string;
+    pub: RemoteAsset;
+    plan: AssetPlan;
+    dest: string;
+    opts: Pick<OpenOptions, "strict" | "keepCompressed">;
+    log: (line: string) => void;
+    progress?: ProgressSink;
+    signal?: AbortSignal;
+}
+export declare function fetchAsset(j: AssetJob): Promise<LandedFile>;
 export declare function land(raw: string, pub: {
     name: string;
     digest: string | null;

package/dist/types/remote.d.ts

@@ -8,6 +8,7 @@ export interface Remote {
     tags(repo: string): Promise<string[]>;
     assets(repo: string, tag: string): Promise<RemoteAsset[]>;
     download(repo: string, tag: string, name: string, dest: string): Promise<void>;
+    bytes?(repo: string, tag: string, name: string, signal?: AbortSignal): Promise<AsyncIterable<Uint8Array>>;
 }
 export declare const ghRemote: Remote;
 export declare function httpsRemote(env?: Readonly<Record<string, string | undefined>>, fetchImpl?: typeof fetch): Remote;

package/dist/types/results.d.ts

@@ -32,6 +32,7 @@ export interface ResultRow {
     school_id: string | null;
     yob: number | null;
     qualifier: string | null;
+    attach_basis?: number;
 }
 export interface ResultAnswer {
     tag: string;
@@ -40,7 +41,7 @@ export interface ResultAnswer {
     rows: ResultRow[];
     truncated: number;
 }
-export declare function queryResults(rel: Release, db: string, q: ResultQuery): ResultAnswer;
+export declare function queryResults(rel: Release, db: string, q: ResultQuery, basis?: boolean): ResultAnswer;
 export declare class ResultFinder {
     private readonly rel;
     constructor(rel: Release);

package/dist/types/strays.d.ts

@@ -0,0 +1,4 @@
+export declare function pidAlive(pid: number): boolean;
+export declare const PART_FILE: RegExp;
+export declare const BUILDING_FILE: RegExp;
+export declare function sweepDead(dir: string, re: RegExp): string[];

package/dist/types/stream.d.ts

@@ -57,6 +57,7 @@ export interface StreamRow extends Partial<StreamCompetition> {
     school_id: string | null;
     dancer_id: number | null;
     dancer_scope: "per-release" | "stable" | null;
+    attach_basis?: number;
     yob: number | null;
     yob_printed: string | null;
     points_printed: string | null;
@@ -72,6 +73,7 @@ export declare function entryRank(section: string): number;
 export interface StreamShape {
     competition?: boolean;
     competitorsPrinted?: boolean;
+    attachBasis?: boolean;
 }
 export declare const scopeDb: (s: StreamScope) => "majors" | "results";
 export declare function streamSql(q: StreamQuery, shape?: StreamShape): {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@finbheara/archive-client",
-	"version": "0.40.0",
+	"version": "0.42.0",
 	"description": "Open a versioned release of an Irish dance results archive: download, verify, decompress and cache it by tag, keep the cache, and resolve feises and schools against it",
 	"keywords": ["irish-dance", "results", "release", "cache", "sqlite"],
 	"license": "MIT",