@tricoteuses/assemblee 3.2.14 → 3.2.15

package/lib/datasets.d.ts

@@ -4,7 +4,7 @@ export interface Dataset {
     filename: string;
     ignoreForWeb?: boolean;
     legislature: Legislature;
-    repairZip?: (dataset: Dataset, dataDir: string) => void;
+    repairZip?: (dataset: Dataset, dataDir: string) => Promise<void> | void;
     structure: DatasetStructure;
     title: string;
     url?: string;

package/lib/open_data/artifacts.d.ts

@@ -0,0 +1,37 @@
+export interface ArtifactVersion {
+    backend: string | null;
+    cdnPop: string | null;
+    contentLength: number | null;
+    etag: string | null;
+    lastModified: string | null;
+    sourceUrl: string;
+}
+export interface ArtifactManifest extends ArtifactVersion {
+    acceptedAt: string;
+    sha256: string;
+}
+export interface DiscoveryOptions {
+    fetch?: typeof fetch;
+    probeCount?: number;
+    timeoutMs?: number;
+}
+export interface DownloadOptions {
+    fetch?: typeof fetch;
+    retries?: number;
+    timeoutMs?: number;
+}
+export declare function versionFromResponse(response: Response, sourceUrl: string): ArtifactVersion;
+export declare function compareArtifactVersions(left: ArtifactVersion, right: ArtifactVersion): number;
+export declare function isStrictlyNewer(candidate: ArtifactVersion, accepted: ArtifactVersion | undefined): boolean;
+export declare function artifactVersionKey(version: ArtifactVersion): string;
+export declare function probeArtifact(url: string, options?: DiscoveryOptions): Promise<ArtifactVersion>;
+export declare function discoverFreshestArtifact(url: string, options?: DiscoveryOptions): Promise<{
+    freshest: ArtifactVersion;
+    observations: ArtifactVersion[];
+}>;
+export declare function downloadArtifact(candidate: ArtifactVersion, partPath: string, options?: DownloadOptions): Promise<{
+    sha256: string;
+    version: ArtifactVersion;
+}>;
+export declare function readManifest(manifestPath: string): Promise<ArtifactManifest | undefined>;
+export declare function writeJsonAtomic(filePath: string, value: unknown): Promise<void>;

package/lib/open_data/artifacts.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/scripts/monitor_open_data_updates.d.ts

@@ -1 +1,22 @@
+import { Dataset } from '../datasets.js';
+export interface MonitoredDataset {
+    category: string;
+    dataset: Dataset;
+    legislature: number;
+    seedSlots: string[];
+}
+interface PendingUpdate {
+    expiresAt: string;
+    pipelineId?: number;
+    versionKey: string;
+}
+export interface DatasetState {
+    observations: string[];
+    pending?: PendingUpdate;
+    seenOrigins: string[];
+}
+export declare function buildMonitoredDatasets(): MonitoredDataset[];
+export declare function isDatasetDue(monitored: MonitoredDataset, datasetState: DatasetState, now: Date): boolean;
+export declare function delayBeforeDeadline(delay: number, deadline: number, now?: number): number;
+export declare function scheduledPipelineIsStale(maxStartDelayMinutes: number, pipelineSource?: string | undefined, pipelineCreatedAt?: string | undefined, now?: number): boolean;
 export {};

package/lib/scripts/monitor_open_data_updates.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/scripts/retrieve_open_data.d.ts

@@ -1,3 +1,4 @@
+import { z } from 'zod';
 import { Dataset } from '../datasets.js';
 type ProcessDatasetArguments = {
     dataset: Dataset;
@@ -5,90 +6,108 @@ type ProcessDatasetArguments = {
     options: Options;
 };
 declare const retrieveOpenDataParser: import('zod-opts').Parser<{
+    dataset: {
+        description: string;
+        type: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    };
     fetch: {
         alias: string;
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodBoolean>;
+        type: z.ZodDefault<z.ZodBoolean>;
+    };
+    "manifest-dir": {
+        description: string;
+        type: z.ZodOptional<z.ZodString>;
+    };
+    "only-if-new": {
+        description: string;
+        type: z.ZodDefault<z.ZodBoolean>;
+    };
+    "probe-count": {
+        description: string;
+        type: z.ZodDefault<z.ZodNumber>;
     };
     categories: {
         alias: string;
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
+        type: z.ZodDefault<z.ZodArray<z.ZodString>>;
     };
     legislature: {
         alias: string;
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodNumber>>;
+        type: z.ZodDefault<z.ZodArray<z.ZodNumber>>;
     };
     silent: {
         alias: string;
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodBoolean>;
+        type: z.ZodDefault<z.ZodBoolean>;
     };
     verbose: {
         alias: string;
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodBoolean>;
+        type: z.ZodDefault<z.ZodBoolean>;
     };
     "only-recent": {
         description: string;
-        type: import('zod').ZodOptional<import('zod').ZodNumber>;
+        type: z.ZodOptional<z.ZodNumber>;
     };
     skipReindent: {
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodBoolean>;
+        type: z.ZodDefault<z.ZodBoolean>;
     };
     keepDir: {
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodBoolean>;
+        type: z.ZodDefault<z.ZodBoolean>;
     };
     clone: {
         alias: string;
         description: string;
-        type: import('zod').ZodOptional<import('zod').ZodString>;
+        type: z.ZodOptional<z.ZodString>;
     };
     commit: {
         alias: string;
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodBoolean>;
+        type: z.ZodDefault<z.ZodBoolean>;
     };
     remote: {
         alias: string;
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
+        type: z.ZodDefault<z.ZodArray<z.ZodString>>;
     };
     pull: {
         alias: string;
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodBoolean>;
+        type: z.ZodDefault<z.ZodBoolean>;
     };
     fetchDocuments: {
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodBoolean>;
+        type: z.ZodDefault<z.ZodBoolean>;
     };
     parseDocuments: {
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodBoolean>;
+        type: z.ZodDefault<z.ZodBoolean>;
     };
     format: {
         alias: string;
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
+        type: z.ZodDefault<z.ZodArray<z.ZodString>>;
     };
     fetchVideos: {
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodBoolean>;
+        type: z.ZodDefault<z.ZodBoolean>;
     };
     fetchCrCommissions: {
         description: string;
-        type: import('zod').ZodDefault<import('zod').ZodBoolean>;
+        type: z.ZodDefault<z.ZodBoolean>;
     };
 }, [{
     description: string;
-    type: import('zod').ZodString;
+    type: z.ZodString;
     name: "dataDir";
 }]>;
 type Options = ReturnType<typeof retrieveOpenDataParser.parse>;
+export declare function publishDataset(stagedDataPath: string, dataPath: string, partZipPath: string | undefined, zipPath: string): Promise<void>;
+export declare function recoverPublication(dataPath: string): Promise<void>;
 /**
  * Processes a dataset by fetching, unzipping, and converting files from XML to JSON format.
  *
@@ -97,5 +116,5 @@ type Options = ReturnType<typeof retrieveOpenDataParser.parse>;
  * @param {any} options - Options to control the processing behavior, such as fetching and silent mode.
  * @returns {Promise<void>} A promise that resolves when the dataset processing is complete.
  */
-export declare function processDataset({ dataset, dataDir, options: { fetch, silent, skipReindent }, }: ProcessDatasetArguments): Promise<void>;
+export declare function processDataset({ dataset, dataDir, options, }: ProcessDatasetArguments): Promise<void>;
 export {};

package/lib/scripts/retrieve_open_data.test.d.ts

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/assemblee",
-  "version": "3.2.14",
+  "version": "3.2.15",
   "description": "Retrieve, clean up & handle French Assemblée nationale's open data",
   "keywords": [
     "Assemblée nationale",