@tricoteuses/senat 2.22.11 → 2.22.13

package/lib/scripts/retrieve_videos.js

@@ -0,0 +1,461 @@
+// scripts/retrieve_senat_videos_from_agendas.ts
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import fsp from "fs/promises";
+import path from "path";
+import * as cheerio from "cheerio";
+import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendas } from "../loaders";
+import { getSessionsFromStart } from "../types/sessions";
+import { commonOptions } from "./shared/cli_helpers";
+import { getAgendaSegmentTimecodes } from "../utils/nvs-timecode";
+import { decodeHtmlEntities } from "../utils/string_cleaning";
+import { dice, normalize, scoreVideo } from "../utils/scoring";
+import { epochToParisDateTime, toFRDate, toTargetEpoch } from "../utils/date";
+// ===================== Constants =====================
+const MATCH_THRESHOLD = 0.5;
+const MAX_CANDIDATES = 15;
+const STATS = { total: 0, accepted: 0 };
+const VIDEOS_ROOT_FOLDER = "videos";
+const SENAT_VIDEOS_SEARCH_AJAX = "https://videos.senat.fr/senat_videos_search.php";
+const SENAT_DATAS_ROOT = "https://videos.senat.fr/Datas/senat";
+// ===================== CLI =====================
+const optionsDefinitions = [...commonOptions];
+const options = commandLineArgs(optionsDefinitions);
+// ===================== Utils =====================
+async function fetchText(url) {
+    const res = await fetch(url);
+    if (!res.ok)
+        return null;
+    return await res.text();
+}
+async function fetchBuffer(url) {
+    const res = await fetch(url);
+    if (!res.ok)
+        return null;
+    const ab = await res.arrayBuffer();
+    return Buffer.from(ab);
+}
+async function writeIfChanged(p, content) {
+    const exists = await fs.pathExists(p);
+    if (exists) {
+        const old = await fsp.readFile(p, "utf-8");
+        if (old === content)
+            return;
+    }
+    await fsp.writeFile(p, content, "utf-8");
+}
+function queryString(obj) {
+    return Object.entries(obj)
+        .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
+        .join("&");
+}
+function extractCandidatesFromSearchHtml(html) {
+    const $ = cheerio.load(html);
+    const out = [];
+    const re = /video\.(\d+)_([a-z0-9]+)/i;
+    $('h3.card-title a.stretched-link[href*="video."]').each((_, a) => {
+        const href = $(a).attr("href") || "";
+        const m = href.match(re);
+        if (!m)
+            return;
+        const id = m[1];
+        const hash = m[2];
+        const pageUrl = `https://videos.senat.fr/video.${id}_${hash}.html`;
+        const title = ($(a).attr("title") || $(a).text() || "").replace(/\s+/g, " ").trim() || undefined;
+        const isSeancePublique = title?.toLowerCase().includes("séance publique") ?? false;
+        out.push({ id, hash, pageUrl, title, isSeancePublique });
+    });
+    // dedupe
+    const seen = new Set();
+    return out.filter((c) => {
+        const k = `${c.id}_${c.hash}`;
+        if (seen.has(k))
+            return false;
+        seen.add(k);
+        return true;
+    });
+}
+function parseDataNvs(nvs) {
+    const epochStr = nvs.match(/<metadata\s+name="date"\s+value="(\d+)"/i)?.[1];
+    const epoch = epochStr ? Number(epochStr) : undefined;
+    // There can be multiple organes for one video in meta
+    const organes = [];
+    const organesRegex = /<metadata\b[^>]*\bname="organes"[^>]*>/gi;
+    let m;
+    while ((m = organesRegex.exec(nvs)) !== null) {
+        const tag = m[0];
+        const label = tag.match(/\blabel="([^"]+)"/i)?.[1];
+        if (label) {
+            const decoded = decodeHtmlEntities(label).trim();
+            if (decoded)
+                organes.push(decoded);
+        }
+    }
+    if (organes.length === 0) {
+        organes.push("Séance publique");
+    }
+    const firstChapterLabelMatch = nvs.match(/<chapter\b[^>]*\blabel="([^"]+)"/i);
+    const firstChapterLabel = firstChapterLabelMatch ? decodeHtmlEntities(firstChapterLabelMatch[1]).trim() : undefined;
+    return { epoch, organes, firstChapterLabel };
+}
+function buildSenatVodMasterM3u8FromNvs(nvsText) {
+    // serverfiles://senat/2025/10/encoder10_20251022084451_2.mp4
+    const m = nvsText.match(/serverfiles:\/\/senat\/(\d{4})\/(\d{2})\/(encoder\d+)_([0-9]{14})/i);
+    if (!m)
+        return null;
+    const [, yyyy, mm, encoder, stamp] = m;
+    const base = `https://vodsenat.akamaized.net/senat/${yyyy}/${mm}/${encoder}_${stamp}`;
+    return `${base}.smil/master.m3u8`;
+}
+function isAmbiguousTimeOriginal(timeOriginal) {
+    if (!timeOriginal)
+        return false;
+    const s = timeOriginal.toLowerCase();
+    // Catches "14h", "14 h", "14h30", "14 h 30", "14 heures", "14 heure"
+    const timeRe = /\b([01]?\d|2[0-3])\s*(?:h|heures?|heure)\s*(?:([0-5]\d))?\b/g;
+    const times = new Set();
+    let m;
+    while ((m = timeRe.exec(s))) {
+        const hh = String(m[1]).padStart(2, "0");
+        const mm = m[2] ? String(m[2]).padStart(2, "0") : "00";
+        times.add(`${hh}:${mm}`);
+    }
+    // "midi" / "minuit"
+    if (/\bmidi\b/.test(s))
+        times.add("12:00");
+    if (/\bminuit\b/.test(s))
+        times.add("00:00");
+    if (times.size >= 2)
+        return true;
+    const hasDayPeriod = /\b(matin|après-?midi|soir|nuit|journée|toute la journée)\b/.test(s);
+    const hasLinking = /,|\bet\b|\bou\b|\bpuis\b/.test(s);
+    if (times.size === 1 && hasDayPeriod && hasLinking)
+        return true;
+    return false;
+}
+function getAgendaType(agenda) {
+    const o = agenda.organe || "";
+    if (/séance publique/i.test(o))
+        return "Séance publique";
+    return "Commission";
+}
+async function fetchAllSearchPages(args, maxPages = 3) {
+    const pages = [];
+    for (let p = 1; p <= maxPages; p++) {
+        const url = `${SENAT_VIDEOS_SEARCH_AJAX}?${queryString({ ...args, page: String(p) })}`;
+        const html = await fetchText(url);
+        if (!html)
+            break;
+        pages.push(html);
+        if (!/href="\/?video\.\d+_[a-z0-9]+\./i.test(html))
+            break;
+    }
+    return pages;
+}
+function getOrgKey(norm) {
+    if (!norm)
+        return "autre";
+    if (norm.includes("seance publique"))
+        return "seance_publique";
+    if (norm.includes("culture"))
+        return "culture";
+    if (norm.includes("finances"))
+        return "finances";
+    if (norm.includes("sociales"))
+        return "affaires_sociales";
+    if (norm.includes("economiques"))
+        return "affaires_economiques";
+    if (norm.includes("europeennes"))
+        return "affaires_europeennes";
+    if (norm.includes("etrangeres") || norm.includes("forces armees") || norm.includes("defense")) {
+        return "affaires_etrangeres_defense";
+    }
+    if (norm.includes("territoire") || norm.includes("durable")) {
+        return "amenagement_territoire_dd";
+    }
+    if (norm.includes("commission des lois"))
+        return "lois";
+    if (norm.includes("delegation aux collectivites territoriales") || norm.includes("delegation a la decentralisation"))
+        return "delegation_collectivites";
+    if (norm.includes("delegation aux droits des femmes") ||
+        norm.includes("egalite des chances entre les hommes et les femmes"))
+        return "delegation_droits_femmes";
+    if (norm.includes("delegation aux entreprises"))
+        return "delegation_entreprises";
+    if (norm.includes("delegation senatoriale aux outre mer") || norm.includes("delegation aux outre mer"))
+        return "delegation_outre_mer";
+    if (norm.includes("delegation a la prospective"))
+        return "delegation_prospective";
+    if (norm.includes("office parlementaire d evaluation des choix scientifiques et technologiques") ||
+        norm.includes("opecst"))
+        return "opecst";
+    return "autre";
+}
+async function processGroupedReunion(agenda, session, dataDir) {
+    // 1) GuardRails
+    if (!agenda.captationVideo) {
+        // if (!options["silent"]) console.log(`[skip] ${agenda.uid} captationVideo=false`)
+        return;
+    }
+    if (!agenda.date || !agenda.startTime) {
+        // if (!options["silent"]) console.log(`[skip] ${agenda.uid} date/hour missing`)
+        return;
+    }
+    const agendaTs = toTargetEpoch(agenda.startTime, agenda.date);
+    const now = Date.now();
+    if (agendaTs && agendaTs * 1000 > now) {
+        return;
+    }
+    const reunionUid = agenda.uid;
+    const baseDir = path.join(dataDir, VIDEOS_ROOT_FOLDER, String(session), reunionUid);
+    await fs.ensureDir(baseDir);
+    let skipDownload = false;
+    if (options["only-recent"]) {
+        const now = Date.now();
+        const cutoff = now - options["only-recent"] * 24 * 3600 * 1000;
+        const reunionTs = Date.parse(agenda.date);
+        if (reunionTs < cutoff) {
+            // Check if files already exist
+            const dataNvsPath = path.join(baseDir, "data.nvs");
+            const finalplayerNvsPath = path.join(baseDir, "finalplayer.nvs");
+            if (fs.existsSync(dataNvsPath) && fs.existsSync(finalplayerNvsPath)) {
+                skipDownload = true;
+            }
+        }
+    }
+    let master = null;
+    let dataTxt = null;
+    let finalTxt = null;
+    let accepted = false;
+    if (!skipDownload) {
+        STATS.total++;
+        const searchParams = {
+            search: "true",
+            videotype: getAgendaType(agenda),
+        };
+        if (agenda.date) {
+            const fr = toFRDate(agenda.date);
+            searchParams.period = "custom";
+            searchParams.begin = fr;
+            searchParams.end = fr;
+        }
+        if (agenda.organe) {
+            searchParams.organe = agenda.organe;
+        }
+        const pages = await fetchAllSearchPages(searchParams);
+        if (!pages.length) {
+            if (!options["silent"]) {
+                console.log(`[miss] ${agenda.uid} no candidates (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
+            }
+            return;
+        }
+        const combinedHtml = pages.join("\n<!-- PAGE SPLIT -->\n");
+        const candidates = extractCandidatesFromSearchHtml(combinedHtml).slice(0, MAX_CANDIDATES);
+        if (!candidates.length) {
+            if (!options["silent"]) {
+                console.log(`[miss] ${agenda.uid} no candidates after parse (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
+            }
+            return;
+        }
+        // ==== 2) Enrich via data.nvs + scoring; pick best ====
+        let best = null;
+        const timeAmbigious = isAmbiguousTimeOriginal(agenda.events[0].timeOriginal);
+        if (timeAmbigious) {
+            console.log(`[match] ${agenda.uid} timeOriginal ambiguous => ignoring time scoring: "${agenda.events[0].timeOriginal}"`);
+        }
+        for (const c of candidates) {
+            const dataUrl = `${SENAT_DATAS_ROOT}/${c.id}_${c.hash}/content/data.nvs`;
+            const finalUrl = `${SENAT_DATAS_ROOT}/${c.id}_${c.hash}/content/finalplayer.nvs`;
+            const dataBuf = await fetchBuffer(dataUrl);
+            if (!dataBuf)
+                continue;
+            const meta = parseDataNvs(dataBuf.toString("utf-8"));
+            let sameOrg = false;
+            // If organes are too different, go to next candidates
+            if (agenda.organe && meta.organes?.length) {
+                const agendaOrgNorm = normalize(agenda.organe);
+                const agendaKey = getOrgKey(agendaOrgNorm);
+                let bestDice = 0;
+                let hasSameKey = false;
+                for (const vo of meta.organes) {
+                    const videoOrgNorm = normalize(vo);
+                    const videoKey = getOrgKey(videoOrgNorm);
+                    const d = dice(agendaOrgNorm, videoOrgNorm);
+                    if (videoKey === agendaKey && videoKey !== "autre") {
+                        hasSameKey = true;
+                    }
+                    if (d > bestDice)
+                        bestDice = d;
+                }
+                if (hasSameKey) {
+                    sameOrg = true; // we are sure this is the same org
+                }
+                else if (bestDice < 0.8) {
+                    // if diff org and dice too low we skip
+                    continue;
+                }
+            }
+            let videoTitle = c.title;
+            if (c.isSeancePublique && meta.firstChapterLabel) {
+                videoTitle = meta.firstChapterLabel;
+            }
+            const s = scoreVideo(agenda, agendaTs, sameOrg, videoTitle, meta.epoch, meta.organes, timeAmbigious);
+            if (!best || s > best.score) {
+                best = {
+                    id: c.id,
+                    hash: c.hash,
+                    pageUrl: c.pageUrl,
+                    epoch: meta.epoch,
+                    vtitle: videoTitle,
+                    score: s,
+                    vorgane: meta.organes[0],
+                };
+            }
+        }
+        if (!best) {
+            if (!options["silent"])
+                console.log(`[miss] ${agenda.uid} No candidate found for this reunion`);
+            return;
+        }
+        accepted = best.score >= MATCH_THRESHOLD;
+        if (accepted)
+            STATS.accepted++;
+        if (!options["silent"]) {
+            console.log(`[pick] ${agenda.uid} score=${best.score.toFixed(2)}
+      agenda title="${agenda.titre ?? ""}" agenda organe="${agenda.organe ?? ""}" agenda heure=${agenda.startTime} 
+      best title="${best.vtitle ?? ""}" best organe="${best.vorgane ?? ""}"
+      accepted=${accepted}`);
+        }
+        // ==== 3) Write metadata + NVS of the best candidate (always) ====
+        const bestDt = best?.epoch ? epochToParisDateTime(best.epoch) : null;
+        const metadata = {
+            reunionUid,
+            session,
+            accepted,
+            threshold: MATCH_THRESHOLD,
+            agenda: {
+                date: agenda.date,
+                startTime: agenda.startTime,
+                titre: agenda.titre,
+                organe: agenda.organe ?? undefined,
+                uid: agenda.uid,
+            },
+            best: {
+                id: best.id,
+                hash: best.hash,
+                pageUrl: best.pageUrl,
+                epoch: best.epoch ?? null,
+                date: bestDt?.date ?? null,
+                startTime: bestDt?.startTime ?? null,
+                title: best.vtitle ?? null,
+                score: best.score,
+            },
+        };
+        await writeIfChanged(path.join(baseDir, "metadata.json"), JSON.stringify(metadata, null, 2));
+        const dataUrl = `${SENAT_DATAS_ROOT}/${best.id}_${best.hash}/content/data.nvs`;
+        const finalUrl = `${SENAT_DATAS_ROOT}/${best.id}_${best.hash}/content/finalplayer.nvs`;
+        dataTxt = await fetchText(dataUrl);
+        finalTxt = await fetchText(finalUrl);
+        if (dataTxt)
+            await fsp.writeFile(path.join(baseDir, "data.nvs"), dataTxt, "utf-8");
+        if (finalTxt)
+            await fsp.writeFile(path.join(baseDir, "finalplayer.nvs"), finalTxt, "utf-8");
+        if (dataTxt) {
+            master = buildSenatVodMasterM3u8FromNvs(dataTxt);
+        }
+        else {
+            console.log("Cannot download data nvs");
+        }
+    }
+    else {
+        // Skipped download, but need to read data.nvs for urlVideo
+        try {
+            dataTxt = await fsp.readFile(path.join(baseDir, "data.nvs"), "utf-8");
+            finalTxt = await fsp.readFile(path.join(baseDir, "finalplayer.nvs"), "utf-8");
+            master = buildSenatVodMasterM3u8FromNvs(dataTxt);
+        }
+        catch (e) {
+            console.warn(e);
+        }
+    }
+    // ==== 4) Update agenda file (only if accepted + m3u8) ====
+    if ((accepted || skipDownload) && master) {
+        const agendaJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${agenda.uid}.json`);
+        let timecodeDebutVideo = null;
+        let timecodeFinVideo = null;
+        if (dataTxt && finalTxt) {
+            const agendaKey = agenda.titre || agenda.objet || "";
+            const seg = getAgendaSegmentTimecodes(dataTxt, finalTxt, agendaKey);
+            if (!seg) {
+                console.warn(`[warn] Cannot retrieve agenda segment timecodes from reunion ${reunionUid}`);
+            }
+            else {
+                timecodeDebutVideo = seg.start;
+                timecodeFinVideo = seg.end;
+            }
+        }
+        if (await fs.pathExists(agendaJsonPath)) {
+            const raw = await fsp.readFile(agendaJsonPath, "utf-8");
+            let obj;
+            try {
+                obj = JSON.parse(raw);
+            }
+            catch (e) {
+                console.warn(`[warn] invalid JSON in ${agendaJsonPath}:`, e?.message);
+                obj = null;
+            }
+            if (obj && typeof obj === "object" && !Array.isArray(obj)) {
+                const next = { ...obj, urlVideo: master };
+                if (timecodeDebutVideo != null) {
+                    next.timecodeDebutVideo = timecodeDebutVideo;
+                    next.timecodeFinVideo = timecodeFinVideo;
+                }
+                await writeIfChanged(agendaJsonPath, JSON.stringify(next, null, 2));
+                if (!options["silent"]) {
+                    console.log(`[write] ${agenda.uid} urlVideo ← ${master}` +
+                        (timecodeDebutVideo != null ? ` (timecodeDebutVideo ← ${timecodeDebutVideo}s)` : ""));
+                }
+            }
+            else {
+                console.warn(`[warn] expected an object in ${agendaJsonPath}, got ${Array.isArray(obj) ? "array" : typeof obj}`);
+            }
+        }
+        else {
+            console.warn(`[warn] agenda file not found for update: ${agendaJsonPath}`);
+        }
+    }
+    else {
+        console.warn(`[warn] The video url could not be built for reunion `, reunionUid);
+    }
+}
+async function processAll(dataDir, sessions) {
+    console.log("Process all Agendas and fetch video's url");
+    for (const session of sessions) {
+        for (const { item: agenda } of iterLoadSenatAgendas(dataDir, session)) {
+            try {
+                await processGroupedReunion(agenda, session, dataDir);
+            }
+            catch (e) {
+                console.error(`[error] ${agenda?.uid ?? "unknown-uid"}:`, e?.message || e);
+            }
+        }
+    }
+}
+async function main() {
+    const dataDir = options["dataDir"];
+    assert(dataDir, "Missing argument: data directory");
+    const sessions = getSessionsFromStart(options["fromSession"]);
+    console.time("senat-agendas→videos start processing time");
+    await processAll(dataDir, sessions);
+    console.timeEnd("senat-agendas→videos processing time");
+    const { total, accepted } = STATS;
+    const ratio = total ? ((accepted / total) * 100).toFixed(1) : "0.0";
+    console.log(`[summary] accepted=${accepted} / total=${total} (${ratio}%)`);
+}
+main()
+    .then(() => process.exit(0))
+    .catch((err) => {
+    console.error(err);
+    process.exit(1);
+});

package/lib/scripts/shared/cli_helpers.d.ts

@@ -0,0 +1,95 @@
+export declare const categoriesOption: {
+    alias: string;
+    defaultValue: string[];
+    help: string;
+    multiple: boolean;
+    name: string;
+    type: StringConstructor;
+};
+export declare const dataDirDefaultOption: {
+    defaultOption: boolean;
+    help: string;
+    name: string;
+    type: StringConstructor;
+};
+export declare const fromSessionOption: {
+    defaultValue: number;
+    help: string;
+    name: string;
+    type: NumberConstructor;
+};
+export declare const silentOption: {
+    alias: string;
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const verboseOption: {
+    alias: string;
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const onlyRecentOption: {
+    help: string;
+    name: string;
+    type: NumberConstructor;
+};
+export declare const keepDirOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const cloneOption: {
+    alias: string;
+    help: string;
+    name: string;
+    type: StringConstructor;
+};
+export declare const commitOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const remoteOption: {
+    alias: string;
+    help: string;
+    multiple: boolean;
+    name: string;
+    type: StringConstructor;
+};
+export declare const pullOption: {
+    alias: string;
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const fetchDocumentsOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const parseDocumentsOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const commonOptions: ({
+    defaultOption: boolean;
+    help: string;
+    name: string;
+    type: StringConstructor;
+} | {
+    help: string;
+    name: string;
+    type: NumberConstructor;
+} | {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+} | {
+    alias: string;
+    help: string;
+    name: string;
+    type: StringConstructor;
+})[];

package/lib/scripts/shared/cli_helpers.js

@@ -0,0 +1,91 @@
+export const categoriesOption = {
+    alias: "k",
+    defaultValue: ["All"],
+    help: "categories of datasets to reorganize; default All",
+    multiple: true,
+    name: "categories",
+    type: String,
+};
+export const dataDirDefaultOption = {
+    defaultOption: true,
+    help: "directory containing Sénat open data files",
+    name: "dataDir",
+    type: String,
+};
+export const fromSessionOption = {
+    defaultValue: 2022,
+    help: "session year to retrieve data from; default 2022",
+    name: "fromSession",
+    type: Number,
+};
+export const silentOption = {
+    alias: "s",
+    help: "don't log anything",
+    name: "silent",
+    type: Boolean,
+};
+export const verboseOption = {
+    alias: "v",
+    help: "verbose logs",
+    name: "verbose",
+    type: Boolean,
+};
+export const onlyRecentOption = {
+    help: "retrieve only documents created within the last N days (that are not already downloaded)",
+    name: "only-recent",
+    type: Number,
+};
+export const keepDirOption = {
+    help: "keep directories when cleaning data",
+    name: "keepDir",
+    type: Boolean,
+};
+export const cloneOption = {
+    alias: "C",
+    help: "clone repositories from given group (or organization) git URL",
+    name: "clone",
+    type: String,
+};
+export const commitOption = {
+    help: "commit clean files",
+    name: "commit",
+    type: Boolean,
+};
+export const remoteOption = {
+    alias: "r",
+    help: "push commit to given remote",
+    multiple: true,
+    name: "remote",
+    type: String,
+};
+export const pullOption = {
+    alias: "p",
+    help: "pull repositories before proceeding",
+    name: "pull",
+    type: Boolean,
+};
+export const fetchDocumentsOption = {
+    help: "download documents",
+    name: "fetchDocuments",
+    type: Boolean,
+};
+export const parseDocumentsOption = {
+    help: "parse documents",
+    name: "parseDocuments",
+    type: Boolean,
+};
+export const commonOptions = [
+    categoriesOption,
+    dataDirDefaultOption,
+    fromSessionOption,
+    silentOption,
+    verboseOption,
+    onlyRecentOption,
+    keepDirOption,
+    cloneOption,
+    commitOption,
+    remoteOption,
+    pullOption,
+    fetchDocumentsOption,
+    parseDocumentsOption,
+];

package/lib/scripts/shared/util.d.ts

@@ -0,0 +1,4 @@
+export declare function isOptionEmptyOrHasValue(option: string, value: string): boolean;
+export declare function ensureAndClearDirSync(dir: string): void;
+export declare function ensureAndClearDir(path: string): void;
+export declare function fetchWithRetry(url: string, retries?: number, backoff?: number): Promise<Response>;

package/lib/scripts/shared/util.js

@@ -0,0 +1,35 @@
+import fs from "fs-extra";
+import path from "path";
+export function isOptionEmptyOrHasValue(option, value) {
+    return !option || option.length === 0 || option.includes(value);
+}
+export function ensureAndClearDirSync(dir) {
+    fs.ensureDirSync(dir);
+    for (const name of fs.readdirSync(dir)) {
+        fs.rmSync(path.join(dir, name), { recursive: true, force: true });
+    }
+}
+export function ensureAndClearDir(path) {
+    if (!fs.existsSync(path)) {
+        fs.mkdirSync(path, { recursive: true });
+    }
+    else {
+        fs.emptyDirSync(path);
+    }
+}
+export async function fetchWithRetry(url, retries = 3, backoff = 300) {
+    for (let attempt = 0; attempt < retries; attempt++) {
+        try {
+            return await fetch(url);
+        }
+        catch (error) {
+            if (attempt === retries - 1) {
+                throw error;
+            }
+            console.warn(`Fetch attempt ${attempt + 1} for ${url} failed. Retrying in ${backoff}ms…`);
+            await new Promise((resolve) => setTimeout(resolve, backoff));
+            backoff *= 2;
+        }
+    }
+    throw new Error(`Failed to fetch ${url} after ${retries} attempts`);
+}

package/lib/scripts/test_iter_load.d.ts

@@ -0,0 +1 @@
+export {};