@yglin/tw-env-records 0.0.3 → 0.0.5

package/lib/authentication.d.ts

@@ -0,0 +1,8 @@
+import { AuthenticationService } from '@feathersjs/authentication';
+import type { Application } from './declarations';
+declare module './declarations' {
+    interface ServiceTypes {
+        authentication: AuthenticationService;
+    }
+}
+export declare const authentication: (app: Application) => void;

package/lib/client.d.ts

@@ -7,7 +7,7 @@ export type { AskAi, AskAiData, AskAiQuery, AskAiPatch, AskAiRequest, AskAiRespo
 import './services/tag/tag.shared';
 export type { Tag, TagData, TagQuery, TagPatch } from './services/tag/tag.shared';
 import './services/meta/meta.shared';
-export type { Meta, MetaData, MetaQuery, MetaPatch, ServerInfo, YearlyStatistics, PerCountyStatistics, TagsTopN, PeopleTopN, TagsSimilarityConfigs } from './services/meta/meta.shared';
+export type { Meta, MetaData, MetaQuery, MetaPatch, ServerInfo, YearlyStatistics, PerCountyStatistics, TagsTopN, PeopleTopN, TagsSimilarityConfigs, DatabaseStatistics } from './services/meta/meta.shared';
 import './services/person/person.shared';
 export type { Person, PersonData, PersonQuery, PersonPatch } from './services/person/person.shared';
 import './services/place-names/place-names.shared';

package/lib/configuration.d.ts

@@ -106,36 +106,117 @@ export declare const configurationSchema: import("@sinclair/typebox").TIntersect
             apiKey: import("@sinclair/typebox").TString<string>;
         }>;
         gemini: import("@sinclair/typebox").TObject<{
-            api01: import("@sinclair/typebox").TObject<{
-                model: import("@sinclair/typebox").TString<string>;
-                apiKey01: import("@sinclair/typebox").TString<string>;
-            }>;
-            api02: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
-                model: import("@sinclair/typebox").TString<string>;
-                apiKey02: import("@sinclair/typebox").TString<string>;
-            }>>;
+            apiKey: import("@sinclair/typebox").TString<string>;
+            model: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
         }>;
     }>;
     services: import("@sinclair/typebox").TObject<{
         askAi: import("@sinclair/typebox").TObject<{
-            recordsLimit: import("@sinclair/typebox").TNumber;
+            ai_model: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            ai_api_key: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            recordsLimit: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            maxRequestsPerDay: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            keepDailyStatsDays: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
         }>;
     }>;
-    crawler: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
-        enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
-        schedule: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
-        timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+    jobs: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
         internalSecret: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
-    }>>;
-    analyzer: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
-        enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
-        schedule: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
-        timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
-    }>>;
-    audit: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
-        enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
-        schedule: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
-        timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+        crawler: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+            enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+            schedule: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            internalSecret: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+        }>>;
+        crawlerBatch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+            enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+            timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            schedules: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                submit: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+                process: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+                cleanup: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            }>>;
+            batch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                use_file_input: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+                max_requests_per_batch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                requests_per_batch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                display_name_prefix: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            }>>;
+            processing: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                max_retry_attempts: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                retry_interval_ms: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            }>>;
+            cleanup: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                cleanup_after_days: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                keep_failed_jobs_days: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            }>>;
+            internalSecret: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+        }>>;
+        analyzer: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+            enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+            schedule: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            internalSecret: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+        }>>;
+        audit: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+            enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+            schedule: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            internalSecret: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+        }>>;
+        auditBatch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+            enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+            timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            schedules: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                submit: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+                process: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+                cleanup: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            }>>;
+            batch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                use_file_input: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+                max_requests_per_batch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                requests_per_batch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                display_name_prefix: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            }>>;
+            processing: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                max_retry_attempts: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                retry_interval_ms: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            }>>;
+            cleanup: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                cleanup_after_days: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                keep_failed_jobs_days: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            }>>;
+            internalSecret: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+        }>>;
+        duplicateCheckBatch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+            enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+            timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            max_records_per_window: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            max_candidates_per_record: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            similarity_threshold: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            schedules: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                submit: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+                process: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            }>>;
+            batch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                use_file_input: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+                requests_per_batch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                display_name_prefix: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            }>>;
+            processing: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                max_retry_attempts: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                retry_interval_ms: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                confidence_threshold: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            }>>;
+            merge: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                story_min_length: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            }>>;
+            backup_failure_log: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+                path: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+                retention_days: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            }>>;
+            internalSecret: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+        }>>;
     }>>;
 }>]>;
 export type ApplicationConfiguration = Static<typeof configurationSchema>;

package/lib/google/gemini-client-factory.d.ts

@@ -0,0 +1,72 @@
+import { Application } from '../declarations';
+export interface IGeminiClient {
+    models: {
+        generateContent: (params: any) => Promise<any>;
+    };
+    batches: {
+        create: (params: any) => Promise<any>;
+        get: (params: any) => Promise<any>;
+        list: (params?: any) => Promise<any>;
+    };
+    files: {
+        download: (params: any) => Promise<void>;
+    };
+}
+/**
+ * Default configuration applied to all Gemini requests
+ * NOTE: Google Search grounding is NOT included because the Batch API
+ * does not return grounding metadata, making it ineffective for batch jobs.
+ */
+export declare const GEMINI_DEFAULT_CONFIG: {};
+/**
+ * Default Gemini model
+ */
+export declare const DEFAULT_GEMINI_MODEL = "gemini-2.5-flash";
+declare class GeminiClientFactory {
+    private instance;
+    private mockInstance;
+    private model;
+    /**
+     * Initialize the Gemini client from application configuration
+     * Call this once at application startup
+     */
+    initialize(app: Application): void;
+    /**
+     * Get the singleton Gemini client instance
+     * Throws error if not initialized
+     */
+    getClient(): IGeminiClient;
+    /**
+     * Get the configured model name
+     */
+    getModel(): string;
+    /**
+     * Get default configuration for Gemini API requests
+     * NOTE: Does not include Google Search grounding since Batch API doesn't support it
+     */
+    getDefaultConfig(): typeof GEMINI_DEFAULT_CONFIG;
+    /**
+     * Set a mock client for testing
+     * WARNING: Only use in tests!
+     */
+    setMockClient(mockClient: IGeminiClient | null): void;
+    /**
+     * Set mock model for testing
+     * WARNING: Only use in tests!
+     */
+    setMockModel(model: string): void;
+    /**
+     * Reset the factory (for testing)
+     * WARNING: Only use in tests!
+     */
+    resetForTesting(): void;
+    /**
+     * Check if client is initialized
+     */
+    isInitialized(): boolean;
+}
+export declare const geminiClientFactory: GeminiClientFactory;
+export declare function getGeminiClient(): IGeminiClient;
+export declare function getGeminiModel(): string;
+export declare function getGeminiConfig(): typeof GEMINI_DEFAULT_CONFIG;
+export {};

package/lib/hooks/after-askai-create.d.ts

@@ -0,0 +1,2 @@
+import type { HookContext } from '@feathersjs/feathers';
+export declare const afterAskAiCreate: (context: HookContext) => Promise<HookContext<import("@feathersjs/feathers").Application<any, any>, any>>;

package/lib/hooks/before-askai-create.d.ts

@@ -0,0 +1,2 @@
+import type { HookContext } from '@feathersjs/feathers';
+export declare const beforeAskAiCreate: (context: HookContext) => Promise<HookContext<import("@feathersjs/feathers").Application<any, any>, any>>;

package/lib/internal/analyzer-endpoints.d.ts

@@ -0,0 +1,2 @@
+import { Application } from '../declarations';
+export declare const analyzerEndpoints: (app: Application) => void;

package/lib/internal/audit-batch-endpoints.d.ts

@@ -0,0 +1,2 @@
+import { Application } from '../declarations';
+export declare const auditBatchEndpoints: (app: Application) => void;

package/lib/internal/crawler-batch-endpoints.d.ts

@@ -0,0 +1,2 @@
+import { Application } from '../declarations';
+export declare const crawlerBatchEndpoints: (app: Application) => void;

package/lib/internal/crawler-endpoints.d.ts

@@ -1,2 +1,2 @@
-import { Application } from "../declarations";
+import { Application } from '../declarations';
 export declare const crawlerEndpoints: (app: Application) => void;

package/lib/internal/duplicate-check-batch-endpoints.d.ts

@@ -0,0 +1,2 @@
+import { Application } from '../declarations';
+export declare const duplicateCheckBatchEndpoints: (app: Application) => void;

package/lib/jobs/audit-batch-process.d.ts

@@ -0,0 +1,10 @@
+import { Application } from '../declarations';
+import { GenericBatchProcessStats } from './shared/index';
+export interface ProcessStats extends GenericBatchProcessStats {
+    recordsUpdated: number;
+    recordsSkipped: number;
+}
+/**
+ * Process all pending audit batch jobs
+ */
+export declare function processPendingBatches(app: Application): Promise<ProcessStats>;

package/lib/jobs/audit-batch-state.d.ts

@@ -0,0 +1,42 @@
+export type BatchAuditStats = {
+    submitted?: number;
+    processed?: number;
+    recordsQueued?: number;
+    recordsUpdated?: number;
+    recordsSkipped?: number;
+    cleanedUp?: number;
+    errors?: number;
+};
+export type BatchAuditState = {
+    submitRunning: boolean;
+    processRunning: boolean;
+    lastSubmitAt?: string;
+    lastProcessAt?: string;
+    lastCleanupAt?: string;
+    lastSubmitDurationMs?: number;
+    lastProcessDurationMs?: number;
+    lastSubmitResult?: BatchAuditStats;
+    lastProcessResult?: BatchAuditStats;
+    lastError?: string;
+    pendingJobs?: number;
+    runningJobs?: number;
+    succeededJobs?: number;
+};
+export declare function getBatchAuditState(): BatchAuditState;
+export declare function startSubmit(): void;
+export declare function finishSubmit(result: BatchAuditStats, durationMs: number): void;
+export declare function failSubmit(error: unknown, durationMs: number): void;
+export declare function startProcess(): void;
+export declare function finishProcess(result: BatchAuditStats, durationMs: number): void;
+export declare function failProcess(error: unknown, durationMs: number): void;
+export declare function updateJobCounts(pending: number, running: number, succeeded: number): void;
+export declare function updateCleanupTime(): void;
+/**
+ * Reset state for testing purposes
+ * WARNING: Only use in tests!
+ */
+export declare function resetStateForTesting(): void;
+/**
+ * Reconstruct state from database on service startup
+ */
+export declare function reconstructFromDatabase(batchJobsService: any): Promise<void>;

package/lib/jobs/audit-batch-submit.d.ts

@@ -0,0 +1,13 @@
+import { Application } from '../declarations';
+import { GenericBatchSubmitStats } from './shared/index';
+export interface SubmitStats extends GenericBatchSubmitStats {
+    recordsQueued: number;
+}
+/**
+ * Build the audit prompt for a single record
+ */
+export declare function buildAuditPrompt(record: any): string;
+/**
+ * Submit a new audit batch job
+ */
+export declare function submitAuditBatch(app: Application): Promise<SubmitStats>;

package/lib/jobs/audit-batch.d.ts

@@ -0,0 +1,36 @@
+import { Application } from '../declarations';
+import { type SubmitStats } from './audit-batch-submit';
+import { type ProcessStats } from './audit-batch-process';
+import { getBatchAuditState, reconstructFromDatabase } from './audit-batch-state';
+import { type CleanupStats } from './shared/index';
+export interface CycleStats {
+    submit: SubmitStats;
+    process: ProcessStats;
+    cleanup: CleanupStats;
+}
+/**
+ * Submit phase: Create and submit new audit batch jobs
+ */
+export declare function submitPhase(app: Application): Promise<SubmitStats>;
+/**
+ * Process phase: Poll and process completed audit batch jobs
+ */
+export declare function processPhase(app: Application): Promise<ProcessStats>;
+/**
+ * Cleanup phase: Remove old completed/failed batch jobs
+ */
+export declare function cleanupPhase(app: Application): Promise<CleanupStats>;
+/**
+ * Get current status of audit batch jobs
+ */
+export declare function getAuditBatchStatus(app: Application): Promise<{
+    pending: number;
+    running: number;
+    succeeded: number;
+    failed: number;
+}>;
+/**
+ * Run a complete audit batch cycle (submit + process + cleanup)
+ */
+export declare function runAuditBatchCycle(app: Application): Promise<CycleStats>;
+export { getBatchAuditState, reconstructFromDatabase };

package/lib/jobs/crawler-batch-process.d.ts

@@ -0,0 +1,10 @@
+import { Application } from '../declarations';
+import { GenericBatchProcessStats } from './shared/index';
+export interface ProcessStats extends GenericBatchProcessStats {
+    recordsExtracted?: number;
+    recordsCreated?: number;
+}
+/**
+ * Process all pending crawl batch jobs
+ */
+export declare function processPendingBatches(app: Application): Promise<ProcessStats>;

package/lib/jobs/crawler-batch-state.d.ts

@@ -0,0 +1,70 @@
+export type BatchCrawlerStats = {
+    submitted?: number;
+    processed?: number;
+    recordsExtracted?: number;
+    recordsCreated?: number;
+    recordsUpdated?: number;
+    duplicatesChecked?: number;
+    cleanedUp?: number;
+    errors?: number;
+};
+export type BatchCrawlerState = {
+    submitRunning: boolean;
+    processRunning: boolean;
+    lastSubmitAt?: string;
+    lastProcessAt?: string;
+    lastCleanupAt?: string;
+    lastSubmitDurationMs?: number;
+    lastProcessDurationMs?: number;
+    lastSubmitResult?: BatchCrawlerStats;
+    lastProcessResult?: BatchCrawlerStats;
+    lastError?: string;
+    pendingJobs?: number;
+    runningJobs?: number;
+    succeededJobs?: number;
+};
+/**
+ * Get current crawler batch state
+ */
+export declare function getBatchCrawlerState(): BatchCrawlerState;
+/**
+ * Mark submit phase as started
+ */
+export declare function startSubmit(): void;
+/**
+ * Mark submit phase as completed
+ */
+export declare function finishSubmit(result: BatchCrawlerStats, durationMs: number): void;
+/**
+ * Mark submit phase as failed
+ */
+export declare function failSubmit(error: unknown, durationMs: number): void;
+/**
+ * Mark process phase as started
+ */
+export declare function startProcess(): void;
+/**
+ * Mark process phase as completed
+ */
+export declare function finishProcess(result: BatchCrawlerStats, durationMs: number): void;
+/**
+ * Mark process phase as failed
+ */
+export declare function failProcess(error: unknown, durationMs: number): void;
+/**
+ * Update job counts from database
+ */
+export declare function updateJobCounts(pending: number, running: number, succeeded: number): void;
+/**
+ * Mark cleanup operation with current timestamp
+ */
+export declare function updateCleanupTime(): void;
+/**
+ * Reset state for testing
+ */
+export declare function resetStateForTesting(): void;
+/**
+ * Reconstruct state from database
+ * Used on service startup
+ */
+export declare function reconstructFromDatabase(batchJobsService: any): Promise<void>;

package/lib/jobs/crawler-batch-submit.d.ts

@@ -0,0 +1,8 @@
+import { Application } from '../declarations';
+import { GenericBatchSubmitStats } from './shared/index';
+export interface SubmitStats extends GenericBatchSubmitStats {
+}
+/**
+ * Submit a new crawl batch job
+ */
+export declare function submitCrawlBatch(app: Application): Promise<SubmitStats>;

package/lib/jobs/crawler-batch.d.ts

@@ -0,0 +1,51 @@
+import { Application } from '../declarations';
+import { SubmitStats } from './crawler-batch-submit';
+import { ProcessStats } from './crawler-batch-process';
+import { getBatchCrawlerState, BatchCrawlerState, reconstructFromDatabase } from './crawler-batch-state';
+import { CleanupStats as SharedCleanupStats } from './shared/index';
+export type CleanupStats = SharedCleanupStats;
+export interface CycleStats {
+    submitted: number;
+    processed: number;
+    recordsExtracted: number;
+    recordsCreated: number;
+    recordsUpdated: number;
+    duplicatesChecked: number;
+    cleanedUp: number;
+    errors: number;
+}
+export type { SubmitStats } from './crawler-batch-submit';
+export type { ProcessStats } from './crawler-batch-process';
+export interface BatchStatus {
+    id: number;
+    job_name: string;
+    job_type: string;
+    status: string;
+    model: string;
+    request_count: number;
+    created_at: string;
+    updated_at: string;
+    completed_at?: string;
+    error_message?: string;
+}
+/**
+ * Submit phase - create and submit new batch jobs
+ */
+export declare function submitPhase(app: Application): Promise<SubmitStats>;
+/**
+ * Process phase - poll and process completed batch jobs
+ */
+export declare function processPhase(app: Application): Promise<ProcessStats>;
+/**
+ * Cleanup phase - remove old completed jobs based on retention policy
+ */
+export declare function cleanupPhase(app: Application): Promise<CleanupStats>;
+/**
+ * Get status of all batch jobs
+ */
+export declare function getBatchStatus(app: Application): Promise<BatchStatus[]>;
+/**
+ * Run full batch crawl cycle (submit → process → cleanup)
+ */
+export declare function runBatchCrawlCycle(app: Application): Promise<CycleStats>;
+export { getBatchCrawlerState, BatchCrawlerState, reconstructFromDatabase };

package/lib/jobs/crawler-prompt.d.ts

@@ -0,0 +1,6 @@
+import { Application } from '../declarations';
+/**
+ * Build the crawl prompt for batch request
+ * Reuses the same prompt logic as the synchronous crawler
+ */
+export declare function buildCrawlPrompt(app: Application): Promise<string>;

package/lib/jobs/crawler-state.d.ts

@@ -1,4 +1,4 @@
-import { CrawlStats } from "./crawler";
+import { CrawlStats } from './crawler';
 export type CrawlerState = {
     running: boolean;
     lastRunAt?: string;
@@ -10,3 +10,8 @@ export declare function getCrawlerState(): CrawlerState;
 export declare function startRun(): void;
 export declare function finishRun(result: CrawlStats, durationMs: number): void;
 export declare function failRun(error: unknown, durationMs: number): void;
+/**
+ * Reset state for testing purposes
+ * WARNING: Only use in tests!
+ */
+export declare function resetStateForTesting(): void;

package/lib/jobs/crawler.d.ts

@@ -1,4 +1,4 @@
-import { Application } from "../declarations";
+import { Application } from '../declarations';
 export type CrawlStats = {
     created: number;
     updated: number;

package/lib/jobs/duplicate-check/find-candidates.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Find records needing duplicate check and their potential duplicate candidates
+ */
+import { Application } from '../../declarations';
+/**
+ * Find records that need duplicate checking
+ * @param app - Feathers application
+ * @param limit - Maximum number of records to return (default: 50)
+ * @returns Array of records with duplicate_check_status = 'pending'
+ */
+export declare function findRecordsNeedingDuplicateCheck(app: Application, limit?: number): Promise<any[]>;
+/**
+ * Calculate Jaccard similarity between two arrays
+ * @param arr1 - First array
+ * @param arr2 - Second array
+ * @returns Similarity score between 0.0 and 1.0
+ */
+export declare function calculateJaccardSimilarity(arr1: string[], arr2: string[]): number;
+/**
+ * Find potential duplicate candidates for a given record
+ * Uses two-phase filtering: database filter (date + place) + in-memory similarity (tags + people)
+ *
+ * @param app - Feathers application
+ * @param record - The record to find duplicates for
+ * @param maxCandidates - Maximum number of candidates to return (default: 20)
+ * @param similarityThreshold - Minimum Jaccard similarity threshold (default: 0.5)
+ * @returns Array of candidate records that might be duplicates
+ */
+export declare function findDuplicateCandidates(app: Application, record: any, maxCandidates?: number, similarityThreshold?: number): Promise<any[]>;

package/lib/jobs/duplicate-check/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Duplicate-check module exports
+ */
+export { findRecordsNeedingDuplicateCheck, findDuplicateCandidates, calculateJaccardSimilarity } from './find-candidates';
+export { buildDuplicateCheckPrompt } from './prompt';
+export { mergeRecordsFromGeminiResponse, type MergeResult } from './merge-records';

package/lib/jobs/duplicate-check/log-failed-backup.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Log failed MongoDB backup to file for recovery
+ */
+/**
+ * Log a failed MongoDB backup to file
+ * Writes in JSONL format (one JSON object per line) for easy recovery
+ *
+ * @param record - The record that failed to backup
+ * @param error - The error that occurred
+ * @param metadata - Additional metadata about the deletion
+ */
+export declare function logFailedMongoBackup(record: any, error: Error, metadata: {
+    merged_into_id?: number;
+    deletion_reason?: string;
+    deleted_at: string;
+    deleted_by?: string;
+}): void;

package/lib/jobs/duplicate-check/merge-records.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Merge duplicate records and handle deletion with MongoDB backup
+ */
+import { Application } from '../../declarations';
+export interface MergeResult {
+    mergedRecordId: number;
+    deletedRecordIds: number[];
+    confidence: number;
+}
+/**
+ * Merge duplicate records into a single record
+ * Creates a new merged record and deletes the duplicates (with MongoDB backup via hook)
+ *
+ * @param app - Feathers application
+ * @param duplicateIds - Array of record IDs to merge
+ * @param mergedRecordData - Data for the merged record
+ * @param confidence - AI confidence score
+ * @returns MergeResult with merged record ID and deleted IDs
+ */
+export declare function mergeRecordsFromGeminiResponse(app: Application, duplicateIds: number[], mergedRecordData: any, confidence: number): Promise<MergeResult>;

package/lib/jobs/duplicate-check/prompt.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Build prompts for Gemini API to detect and merge duplicate records
+ */
+/**
+ * Build a duplicate-check prompt for Gemini API
+ * Asks the AI to identify which candidates are duplicates and provide a merged record
+ *
+ * @param newRecord - The record being checked for duplicates
+ * @param candidates - Array of potential duplicate records
+ * @returns Prompt string for Gemini API
+ */
+export declare function buildDuplicateCheckPrompt(newRecord: any, candidates: any[]): string;