anz-legislation 1.2.1

package/dist/utils/streaming.js

@@ -0,0 +1,333 @@
+/**
+ * Streaming Utilities
+ *
+ * Provides streaming support for large data exports to minimize memory usage.
+ * Allows exporting GBs of data without running out of memory.
+ */
+import { createWriteStream } from 'fs';
+import { Readable } from 'stream';
+import { pipeline } from 'stream/promises';
+import { searchWorks, getWork } from '@client';
+/**
+ * Stream exporter with progress tracking
+ */
+export class StreamExporter {
+    outputPath;
+    format;
+    includeMetadata;
+    batchSize;
+    maxResults;
+    concurrency;
+    writeStream;
+    bytesWritten = 0;
+    processed = 0;
+    aborted = false;
+    constructor(options) {
+        this.outputPath = options.outputPath;
+        this.format = options.format;
+        this.includeMetadata = options.includeMetadata ?? false;
+        this.batchSize = options.batchSize ?? 100;
+        this.maxResults = options.maxResults ?? Infinity;
+        this.concurrency = options.concurrency ?? 3;
+    }
+    /**
+     * Export search results to file using streaming
+     */
+    async export(params, onProgress) {
+        const startTime = Date.now();
+        // Create write stream
+        this.writeStream = createWriteStream(this.outputPath, {
+            encoding: 'utf-8',
+            highWaterMark: 64 * 1024, // 64KB chunks
+        });
+        try {
+            // Write header based on format
+            if (this.format === 'csv') {
+                this.writeCsvHeader();
+            }
+            else if (this.format === 'json') {
+                this.writeStream.write('[\n');
+            }
+            // NDJSON doesn't need a header
+            let offset = 0;
+            let totalFetched = 0;
+            let hasMore = true;
+            while (hasMore && !this.aborted) {
+                // Fetch batch
+                const results = await searchWorks({
+                    ...params,
+                    limit: this.batchSize,
+                    offset,
+                });
+                if (results.results.length === 0) {
+                    hasMore = false;
+                    break;
+                }
+                // Process and write each result
+                for (const work of results.results) {
+                    if (this.aborted) {
+                        break;
+                    }
+                    if (totalFetched >= this.maxResults) {
+                        hasMore = false;
+                        break;
+                    }
+                    this.writeWork(work, totalFetched === 0 && this.format === 'json');
+                    this.processed++;
+                    totalFetched++;
+                    // Progress callback
+                    if (onProgress) {
+                        onProgress({
+                            processed: this.processed,
+                            total: results.total,
+                            bytesWritten: this.bytesWritten,
+                            percent: results.total > 0 ? Math.round((this.processed / results.total) * 100) : 0,
+                            estimatedTimeRemaining: this.calculateETA(startTime, this.processed),
+                        });
+                    }
+                }
+                offset += this.batchSize;
+            }
+            // Write footer based on format
+            if (this.format === 'json') {
+                this.writeStream.write('\n]');
+            }
+            // Include metadata if requested
+            if (this.includeMetadata && this.format !== 'ndjson') {
+                this.writeMetadata(params, totalFetched);
+            }
+            // Close stream
+            await this.closeStream();
+            return {
+                processed: this.processed,
+                bytesWritten: this.bytesWritten,
+            };
+        }
+        catch (error) {
+            this.writeStream.destroy();
+            throw error;
+        }
+    }
+    /**
+     * Stream work details for multiple works
+     */
+    async exportWorkDetails(workIds, outputPath, format = 'json', onProgress) {
+        const writeStream = createWriteStream(outputPath, {
+            encoding: 'utf-8',
+            highWaterMark: 64 * 1024,
+        });
+        const startTime = Date.now();
+        let processed = 0;
+        let bytesWritten = 0;
+        try {
+            if (format === 'json') {
+                writeStream.write('[\n');
+            }
+            for (let i = 0; i < workIds.length; i++) {
+                const workId = workIds[i];
+                try {
+                    const work = await getWork(workId);
+                    const line = format === 'json'
+                        ? (i > 0 ? ',\n' : '') + JSON.stringify(work, null, 2)
+                        : JSON.stringify(work) + '\n';
+                    writeStream.write(line);
+                    bytesWritten += line.length;
+                    processed++;
+                    if (onProgress) {
+                        onProgress({
+                            processed,
+                            total: workIds.length,
+                            bytesWritten,
+                            percent: Math.round((processed / workIds.length) * 100),
+                            estimatedTimeRemaining: this.calculateETA(startTime, processed),
+                        });
+                    }
+                }
+                catch (error) {
+                    // Log error but continue
+                    console.error(`Failed to fetch work ${workId}:`, error.message);
+                }
+                // Rate limiting - small delay between requests
+                if (i < workIds.length - 1) {
+                    await new Promise(resolve => setTimeout(resolve, 100));
+                }
+            }
+            if (format === 'json') {
+                writeStream.write('\n]');
+            }
+            await new Promise((resolve, reject) => {
+                writeStream.end(resolve);
+                writeStream.on('error', reject);
+            });
+            return { processed, bytesWritten };
+        }
+        catch (error) {
+            writeStream.destroy();
+            throw error;
+        }
+    }
+    /**
+     * Abort the streaming export
+     */
+    abort() {
+        this.aborted = true;
+    }
+    /**
+     * Write CSV header
+     */
+    writeCsvHeader() {
+        const header = 'id,title,shortTitle,type,status,date,url,versionCount\n';
+        this.writeStream.write(header);
+        this.bytesWritten += header.length;
+    }
+    /**
+     * Write work to stream
+     */
+    writeWork(work, isFirst) {
+        let line = '';
+        switch (this.format) {
+            case 'csv':
+                line = this.formatCsvRow(work);
+                break;
+            case 'json':
+                line = (isFirst ? '' : ',\n') + JSON.stringify(work, null, 2);
+                break;
+            case 'ndjson':
+                line = JSON.stringify(work) + '\n';
+                break;
+        }
+        this.writeStream.write(line);
+        this.bytesWritten += line.length;
+    }
+    /**
+     * Format work as CSV row
+     */
+    formatCsvRow(work) {
+        // Escape CSV fields
+        const escape = (str) => {
+            if (str.includes(',') || str.includes('"') || str.includes('\n')) {
+                return `"${str.replace(/"/g, '""')}"`;
+            }
+            return str;
+        };
+        return ([
+            work.id,
+            escape(work.title),
+            work.shortTitle ? escape(work.shortTitle) : '',
+            work.type,
+            work.status,
+            work.date,
+            work.url,
+            work.versionCount.toString(),
+        ].join(',') + '\n');
+    }
+    /**
+     * Write metadata at end of file
+     */
+    writeMetadata(params, totalResults) {
+        const metadata = {
+            _metadata: {
+                exportedAt: new Date().toISOString(),
+                query: params.query,
+                totalResults,
+                format: this.format,
+            },
+        };
+        if (this.format === 'csv') {
+            this.writeStream.write('\n# Metadata\n');
+            this.writeStream.write(`# Exported: ${metadata._metadata.exportedAt}\n`);
+            this.writeStream.write(`# Query: ${params.query}\n`);
+            this.writeStream.write(`# Total: ${totalResults}\n`);
+        }
+        else {
+            this.writeStream.write(',\n');
+            this.writeStream.write(JSON.stringify(metadata, null, 2));
+        }
+        this.bytesWritten += JSON.stringify(metadata).length;
+    }
+    /**
+     * Close write stream
+     */
+    async closeStream() {
+        return new Promise((resolve, reject) => {
+            this.writeStream.end(() => {
+                resolve();
+            });
+            this.writeStream.on('error', reject);
+        });
+    }
+    /**
+     * Calculate estimated time remaining
+     */
+    calculateETA(startTime, processed) {
+        if (processed === 0) {
+            return undefined;
+        }
+        const elapsed = Date.now() - startTime;
+        const avgTimePerItem = elapsed / processed;
+        const remaining = this.maxResults - processed;
+        return Math.round(avgTimePerItem * remaining);
+    }
+}
+/**
+ * Create a readable stream from paginated API results
+ */
+export function createPaginatedStream(params, options = {}) {
+    const batchSize = options.batchSize ?? 100;
+    let offset = 0;
+    let hasMore = true;
+    let pending = null;
+    return new Readable({
+        objectMode: true,
+        read() {
+            if (pending) {
+                return;
+            }
+            if (!hasMore) {
+                this.push(null);
+                return;
+            }
+            pending = (async () => {
+                try {
+                    const results = await searchWorks({
+                        ...params,
+                        limit: batchSize,
+                        offset,
+                    });
+                    if (results.results.length === 0) {
+                        hasMore = false;
+                        this.push(null);
+                        return;
+                    }
+                    for (const work of results.results) {
+                        this.push(work);
+                    }
+                    offset += batchSize;
+                }
+                catch (error) {
+                    this.destroy(error);
+                }
+                finally {
+                    pending = null;
+                }
+            })();
+        },
+    });
+}
+/**
+ * Stream processing pipeline
+ *
+ * Example usage:
+ * ```typescript
+ * const stream = createPaginatedStream({ query: 'health' });
+ * const transformed = stream.pipe(new Transform({
+ *   objectMode: true,
+ *   transform(work, encoding, callback) {
+ *     callback(null, { id: work.work_id, title: work.title });
+ *   }
+ * }));
+ *
+ * await pipeline(stream, transformed, createWriteStream('output.json'));
+ * ```
+ */
+export { pipeline };

package/dist/utils/validation.d.ts

@@ -0,0 +1,190 @@
+/**
+ * Input validation schemas using Zod
+ * Provides runtime validation for all user inputs
+ */
+import { z } from 'zod';
+/**
+ * Date string in YYYY-MM-DD format
+ */
+export declare const DateStringSchema: z.ZodEffects<z.ZodString, string, string>;
+/**
+ * Work ID validation for current API work IDs (e.g., act_public_1989_18)
+ */
+export declare const WorkIdSchema: z.ZodString;
+/**
+ * Search query validation
+ */
+export declare const SearchQuerySchema: z.ZodString;
+/**
+ * Legislation type validation
+ */
+export declare const LegislationTypeSchema: z.ZodOptional<z.ZodEnum<["act", "bill", "regulation", "instrument"]>>;
+/**
+ * Legislation status validation
+ */
+export declare const LegislationStatusSchema: z.ZodOptional<z.ZodEnum<["in-force", "not-yet-in-force", "repealed", "partially-repealed", "withdrawn"]>>;
+/**
+ * Output format validation
+ */
+export declare const OutputFormatSchema: z.ZodDefault<z.ZodEnum<["table", "json", "csv"]>>;
+/**
+ * Pagination limit validation
+ */
+export declare const LimitSchema: z.ZodEffects<z.ZodDefault<z.ZodString>, number, string | undefined>;
+/**
+ * Pagination offset validation
+ */
+export declare const OffsetSchema: z.ZodEffects<z.ZodDefault<z.ZodString>, number, string | undefined>;
+/**
+ * Search parameters schema
+ */
+export declare const SearchParamsSchema: z.ZodObject<{
+    query: z.ZodString;
+    type: z.ZodOptional<z.ZodEnum<["act", "bill", "regulation", "instrument"]>>;
+    status: z.ZodOptional<z.ZodEnum<["in-force", "not-yet-in-force", "repealed", "partially-repealed", "withdrawn"]>>;
+    from: z.ZodOptional<z.ZodEffects<z.ZodString, string, string>>;
+    to: z.ZodOptional<z.ZodEffects<z.ZodString, string, string>>;
+    limit: z.ZodEffects<z.ZodDefault<z.ZodString>, number, string | undefined>;
+    offset: z.ZodEffects<z.ZodDefault<z.ZodString>, number, string | undefined>;
+    format: z.ZodDefault<z.ZodEnum<["table", "json", "csv"]>>;
+}, "strip", z.ZodTypeAny, {
+    format: "json" | "table" | "csv";
+    offset: number;
+    limit: number;
+    query: string;
+    type?: "act" | "bill" | "regulation" | "instrument" | undefined;
+    status?: "not-yet-in-force" | "in-force" | "repealed" | "withdrawn" | "partially-repealed" | undefined;
+    from?: string | undefined;
+    to?: string | undefined;
+}, {
+    query: string;
+    type?: "act" | "bill" | "regulation" | "instrument" | undefined;
+    status?: "not-yet-in-force" | "in-force" | "repealed" | "withdrawn" | "partially-repealed" | undefined;
+    format?: "json" | "table" | "csv" | undefined;
+    offset?: string | undefined;
+    limit?: string | undefined;
+    from?: string | undefined;
+    to?: string | undefined;
+}>;
+/**
+ * Get work parameters schema
+ */
+export declare const GetWorkParamsSchema: z.ZodObject<{
+    workId: z.ZodString;
+    versions: z.ZodDefault<z.ZodBoolean>;
+    format: z.ZodDefault<z.ZodEnum<["table", "json", "csv"]>>;
+}, "strip", z.ZodTypeAny, {
+    format: "json" | "table" | "csv";
+    workId: string;
+    versions: boolean;
+}, {
+    workId: string;
+    format?: "json" | "table" | "csv" | undefined;
+    versions?: boolean | undefined;
+}>;
+/**
+ * Citation style validation
+ */
+export declare const CitationStyleSchema: z.ZodDefault<z.ZodEnum<["nzmj", "bibtex", "ris", "enw", "apa"]>>;
+/**
+ * Export parameters schema
+ */
+export declare const ExportParamsSchema: z.ZodObject<{
+    query: z.ZodString;
+    output: z.ZodString;
+    type: z.ZodOptional<z.ZodEnum<["act", "bill", "regulation", "instrument"]>>;
+    status: z.ZodOptional<z.ZodEnum<["in-force", "not-yet-in-force", "repealed", "partially-repealed", "withdrawn"]>>;
+    from: z.ZodOptional<z.ZodEffects<z.ZodString, string, string>>;
+    to: z.ZodOptional<z.ZodEffects<z.ZodString, string, string>>;
+    limit: z.ZodEffects<z.ZodDefault<z.ZodString>, number, string | undefined>;
+}, "strip", z.ZodTypeAny, {
+    limit: number;
+    query: string;
+    output: string;
+    type?: "act" | "bill" | "regulation" | "instrument" | undefined;
+    status?: "not-yet-in-force" | "in-force" | "repealed" | "withdrawn" | "partially-repealed" | undefined;
+    from?: string | undefined;
+    to?: string | undefined;
+}, {
+    query: string;
+    output: string;
+    type?: "act" | "bill" | "regulation" | "instrument" | undefined;
+    status?: "not-yet-in-force" | "in-force" | "repealed" | "withdrawn" | "partially-repealed" | undefined;
+    limit?: string | undefined;
+    from?: string | undefined;
+    to?: string | undefined;
+}>;
+/**
+ * Configuration key validation
+ */
+export declare const ConfigKeySchema: z.ZodEnum<["apiKey", "baseUrl", "timeout", "cacheEnabled", "cacheTTL", "rateLimitPerMinute", "outputFormat", "verbose"]>;
+/**
+ * API URL validation
+ */
+export declare const ApiUrlSchema: z.ZodEffects<z.ZodString, string, string>;
+/**
+ * Timeout validation (in milliseconds)
+ */
+export declare const TimeoutSchema: z.ZodEffects<z.ZodDefault<z.ZodString>, number, string | undefined>;
+/**
+ * Validation error type
+ */
+export interface ValidationError {
+    field: string;
+    message: string;
+    value?: unknown;
+}
+/**
+ * Validate input against schema and return formatted errors
+ */
+export declare function validateInput<T>(schema: z.ZodSchema<T>, data: unknown): {
+    valid: boolean;
+    data?: T;
+    errors?: ValidationError[];
+};
+/**
+ * Sanitize string input to prevent injection attacks
+ */
+export declare function sanitizeInput(input: string): string;
+/**
+ * Search parameters schema (output types for validated data)
+ */
+export interface ValidatedSearchParams {
+    query: string;
+    type?: 'act' | 'bill' | 'regulation' | 'instrument';
+    status?: 'in-force' | 'not-yet-in-force' | 'repealed' | 'partially-repealed' | 'withdrawn';
+    from?: string;
+    to?: string;
+    limit: number;
+    offset: number;
+    format: 'table' | 'json' | 'csv';
+}
+interface ValidationFailure {
+    valid: false;
+    errors: ValidationError[];
+}
+interface SearchValidationSuccess {
+    valid: true;
+    data: z.infer<typeof SearchParamsSchema>;
+}
+interface WorkIdValidationSuccess {
+    valid: true;
+    data: z.infer<typeof WorkIdSchema>;
+}
+interface ExportValidationSuccess {
+    valid: true;
+    data: z.infer<typeof ExportParamsSchema>;
+}
+/**
+ * Validate and sanitize search parameters
+ */
+export declare function validateSearchParams(params: unknown): ValidationFailure | SearchValidationSuccess;
+/**
+ * Validate and sanitize work ID
+ */
+export declare function validateWorkId(workId: string): ValidationFailure | WorkIdValidationSuccess;
+/**
+ * Validate and sanitize export parameters
+ */
+export declare function validateExportParams(params: unknown): ValidationFailure | ExportValidationSuccess;
+export {};