@yglin/tw-env-records 0.0.2 → 0.0.4

package/lib/client.d.ts

@@ -7,7 +7,7 @@ export type { AskAi, AskAiData, AskAiQuery, AskAiPatch, AskAiRequest, AskAiRespo
 import './services/tag/tag.shared';
 export type { Tag, TagData, TagQuery, TagPatch } from './services/tag/tag.shared';
 import './services/meta/meta.shared';
-export type { Meta, MetaData, MetaQuery, MetaPatch, ServerInfo, YearlyStatistics, PerCountyStatistics, TagsTopN, PeopleTopN, TagsSimilarityConfigs } from './services/meta/meta.shared';
+export type { Meta, MetaData, MetaQuery, MetaPatch, ServerInfo, YearlyStatistics, PerCountyStatistics, TagsTopN, PeopleTopN, TagsSimilarityConfigs, DatabaseStatistics } from './services/meta/meta.shared';
 import './services/person/person.shared';
 export type { Person, PersonData, PersonQuery, PersonPatch } from './services/person/person.shared';
 import './services/place-names/place-names.shared';

package/lib/configuration.d.ts

@@ -106,36 +106,84 @@ export declare const configurationSchema: import("@sinclair/typebox").TIntersect
             apiKey: import("@sinclair/typebox").TString<string>;
         }>;
         gemini: import("@sinclair/typebox").TObject<{
-            api01: import("@sinclair/typebox").TObject<{
-                model: import("@sinclair/typebox").TString<string>;
-                apiKey01: import("@sinclair/typebox").TString<string>;
-            }>;
-            api02: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
-                model: import("@sinclair/typebox").TString<string>;
-                apiKey02: import("@sinclair/typebox").TString<string>;
-            }>>;
+            apiKey: import("@sinclair/typebox").TString<string>;
+            model: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
         }>;
     }>;
     services: import("@sinclair/typebox").TObject<{
         askAi: import("@sinclair/typebox").TObject<{
+            ai_model: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            ai_api_key: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
             recordsLimit: import("@sinclair/typebox").TNumber;
         }>;
     }>;
-    crawler: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
-        enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
-        schedule: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
-        timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
-        internalSecret: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
-    }>>;
-    analyzer: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
-        enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
-        schedule: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
-        timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
-    }>>;
-    audit: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
-        enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
-        schedule: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
-        timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+    jobs: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+        crawler: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+            enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+            schedule: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            internalSecret: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+        }>>;
+        crawlerBatch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+            enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+            timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            schedules: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                submit: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+                process: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+                cleanup: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            }>>;
+            batch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                use_file_input: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+                max_requests_per_batch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                requests_per_batch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                display_name_prefix: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            }>>;
+            processing: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                max_retry_attempts: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                retry_interval_ms: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            }>>;
+            cleanup: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                cleanup_after_days: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                keep_failed_jobs_days: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            }>>;
+            internalSecret: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+        }>>;
+        analyzer: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+            enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+            schedule: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            internalSecret: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+        }>>;
+        audit: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+            enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+            schedule: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            internalSecret: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+        }>>;
+        auditBatch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+            enabled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+            timezone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            schedules: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                submit: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+                process: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+                cleanup: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            }>>;
+            batch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                use_file_input: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
+                max_requests_per_batch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                requests_per_batch: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                display_name_prefix: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+            }>>;
+            processing: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                max_retry_attempts: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                retry_interval_ms: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            }>>;
+            cleanup: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
+                cleanup_after_days: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+                keep_failed_jobs_days: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
+            }>>;
+            internalSecret: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString<string>>;
+        }>>;
     }>>;
 }>]>;
 export type ApplicationConfiguration = Static<typeof configurationSchema>;

package/lib/internal/analyzer-endpoints.d.ts

@@ -0,0 +1,2 @@
+import { Application } from '../declarations';
+export declare const analyzerEndpoints: (app: Application) => void;

package/lib/internal/audit-batch-endpoints.d.ts

@@ -0,0 +1,2 @@
+import { Application } from '../declarations';
+export declare const auditBatchEndpoints: (app: Application) => void;

package/lib/internal/crawler-batch-endpoints.d.ts

@@ -0,0 +1,2 @@
+import { Application } from '../declarations';
+export declare const crawlerBatchEndpoints: (app: Application) => void;

package/lib/internal/crawler-endpoints.d.ts

@@ -1,2 +1,2 @@
-import { Application } from "../declarations";
+import { Application } from '../declarations';
 export declare const crawlerEndpoints: (app: Application) => void;

package/lib/jobs/audit-batch-process.d.ts

@@ -0,0 +1,16 @@
+import { Application } from '../declarations';
+import { type IGeminiClient } from './gemini-client-factory';
+export interface ProcessStats {
+    processed: number;
+    recordsUpdated: number;
+    recordsSkipped: number;
+    errors: number;
+}
+/**
+ * Download and parse results from a completed batch job
+ */
+export declare function downloadAndParseResults(client: IGeminiClient, jobName: string, isFileResult: boolean, resultFileName?: string): Promise<any[]>;
+/**
+ * Process all pending audit batch jobs
+ */
+export declare function processPendingBatches(app: Application): Promise<ProcessStats>;

package/lib/jobs/audit-batch-state.d.ts

@@ -0,0 +1,38 @@
+export type BatchAuditStats = {
+    submitted?: number;
+    processed?: number;
+    recordsQueued?: number;
+    recordsUpdated?: number;
+    recordsSkipped?: number;
+    cleanedUp?: number;
+    errors?: number;
+};
+export type BatchAuditState = {
+    submitRunning: boolean;
+    processRunning: boolean;
+    lastSubmitAt?: string;
+    lastProcessAt?: string;
+    lastCleanupAt?: string;
+    lastSubmitDurationMs?: number;
+    lastProcessDurationMs?: number;
+    lastSubmitResult?: BatchAuditStats;
+    lastProcessResult?: BatchAuditStats;
+    lastError?: string;
+    pendingJobs?: number;
+    runningJobs?: number;
+    succeededJobs?: number;
+};
+export declare function getBatchAuditState(): BatchAuditState;
+export declare function startSubmit(): void;
+export declare function finishSubmit(result: BatchAuditStats, durationMs: number): void;
+export declare function failSubmit(error: unknown, durationMs: number): void;
+export declare function startProcess(): void;
+export declare function finishProcess(result: BatchAuditStats, durationMs: number): void;
+export declare function failProcess(error: unknown, durationMs: number): void;
+export declare function updateJobCounts(pending: number, running: number, succeeded: number): void;
+export declare function updateCleanupTime(): void;
+/**
+ * Reset state for testing purposes
+ * WARNING: Only use in tests!
+ */
+export declare function resetStateForTesting(): void;

package/lib/jobs/audit-batch-submit.d.ts

@@ -0,0 +1,21 @@
+import { Application } from '../declarations';
+export interface SubmitStats {
+    submitted: number;
+    recordsQueued: number;
+    jobNames: string[];
+}
+/**
+ * Build the audit prompt for a single record
+ */
+export declare function buildAuditPrompt(record: any): string;
+/**
+ * Create a batch request object for audit
+ */
+/**
+ * Create a batch request object for audit
+ */
+export declare function createBatchRequest(prompt: string, recordId: number): any;
+/**
+ * Submit a new audit batch job
+ */
+export declare function submitAuditBatch(app: Application): Promise<SubmitStats>;

package/lib/jobs/audit-batch.d.ts

@@ -0,0 +1,36 @@
+import { Application } from '../declarations';
+import { type SubmitStats } from './audit-batch-submit';
+import { type ProcessStats } from './audit-batch-process';
+export interface CleanupStats {
+    removed: number;
+}
+export interface CycleStats {
+    submit: SubmitStats;
+    process: ProcessStats;
+    cleanup: CleanupStats;
+}
+/**
+ * Submit phase: Create and submit new audit batch jobs
+ */
+export declare function submitPhase(app: Application): Promise<SubmitStats>;
+/**
+ * Process phase: Poll and process completed audit batch jobs
+ */
+export declare function processPhase(app: Application): Promise<ProcessStats>;
+/**
+ * Cleanup phase: Remove old completed/failed batch jobs
+ */
+export declare function cleanupPhase(app: Application): Promise<CleanupStats>;
+/**
+ * Get current status of audit batch jobs
+ */
+export declare function getAuditBatchStatus(app: Application): Promise<{
+    pending: number;
+    running: number;
+    succeeded: number;
+    failed: number;
+}>;
+/**
+ * Run a complete audit batch cycle (submit + process + cleanup)
+ */
+export declare function runAuditBatchCycle(app: Application): Promise<CycleStats>;

package/lib/jobs/crawler-batch-process.d.ts

@@ -0,0 +1,19 @@
+import { Application } from '../declarations';
+import { type IGeminiClient } from './gemini-client-factory';
+export interface ProcessStats {
+    processed: number;
+    recordsExtracted: number;
+    duplicateCheckJobsCreated: number;
+    recordsCreated: number;
+    recordsUpdated: number;
+    duplicatesChecked: number;
+    errors: number;
+}
+/**
+ * Download and parse results from a completed batch job
+ */
+export declare function downloadAndParseResults(client: IGeminiClient, jobName: string, isFileResult: boolean, resultFileName?: string): Promise<any[]>;
+/**
+ * Process all pending batch jobs
+ */
+export declare function processPendingBatches(app: Application): Promise<ProcessStats>;

package/lib/jobs/crawler-batch-state.d.ts

@@ -0,0 +1,39 @@
+export type BatchCrawlerStats = {
+    submitted?: number;
+    processed?: number;
+    recordsExtracted?: number;
+    recordsCreated?: number;
+    recordsUpdated?: number;
+    duplicatesChecked?: number;
+    cleanedUp?: number;
+    errors?: number;
+};
+export type BatchCrawlerState = {
+    submitRunning: boolean;
+    processRunning: boolean;
+    lastSubmitAt?: string;
+    lastProcessAt?: string;
+    lastCleanupAt?: string;
+    lastSubmitDurationMs?: number;
+    lastProcessDurationMs?: number;
+    lastSubmitResult?: BatchCrawlerStats;
+    lastProcessResult?: BatchCrawlerStats;
+    lastError?: string;
+    pendingJobs?: number;
+    runningJobs?: number;
+    succeededJobs?: number;
+};
+export declare function getBatchCrawlerState(): BatchCrawlerState;
+export declare function startSubmit(): void;
+export declare function finishSubmit(result: BatchCrawlerStats, durationMs: number): void;
+export declare function failSubmit(error: unknown, durationMs: number): void;
+export declare function startProcess(): void;
+export declare function finishProcess(result: BatchCrawlerStats, durationMs: number): void;
+export declare function failProcess(error: unknown, durationMs: number): void;
+export declare function updateJobCounts(pending: number, running: number, succeeded: number): void;
+export declare function updateCleanupTime(): void;
+/**
+ * Reset state for testing purposes
+ * WARNING: Only use in tests!
+ */
+export declare function resetStateForTesting(): void;

package/lib/jobs/crawler-batch-submit.d.ts

@@ -0,0 +1,18 @@
+import { Application } from '../declarations';
+export interface SubmitStats {
+    submitted: number;
+    jobNames: string[];
+}
+/**
+ * Build the crawl prompt for batch request
+ * Reuses the same prompt logic as the synchronous crawler
+ */
+export declare function buildCrawlPrompt(schema: any): string;
+/**
+ * Create a batch request object
+ */
+export declare function createBatchRequest(prompt: string, config: any): any;
+/**
+ * Submit a new crawl batch job
+ */
+export declare function submitCrawlBatch(app: Application): Promise<SubmitStats>;

package/lib/jobs/crawler-batch.d.ts

@@ -0,0 +1,52 @@
+import { Application } from '../declarations';
+import { SubmitStats } from './crawler-batch-submit';
+import { ProcessStats } from './crawler-batch-process';
+import { getBatchCrawlerState, BatchCrawlerState } from './crawler-batch-state';
+export interface CycleStats {
+    submitted: number;
+    processed: number;
+    recordsExtracted: number;
+    recordsCreated: number;
+    recordsUpdated: number;
+    duplicatesChecked: number;
+    cleanedUp: number;
+    errors: number;
+}
+export interface CleanupStats {
+    cleanedUp: number;
+}
+export type { SubmitStats } from './crawler-batch-submit';
+export type { ProcessStats } from './crawler-batch-process';
+export interface BatchStatus {
+    id: number;
+    job_name: string;
+    job_type: string;
+    status: string;
+    model: string;
+    request_count: number;
+    created_at: string;
+    updated_at: string;
+    completed_at?: string;
+    error_message?: string;
+}
+/**
+ * Submit phase - create and submit new batch jobs
+ */
+export declare function submitPhase(app: Application): Promise<SubmitStats>;
+/**
+ * Process phase - poll and process completed batch jobs
+ */
+export declare function processPhase(app: Application): Promise<ProcessStats>;
+/**
+ * Cleanup phase - remove old completed jobs
+ */
+export declare function cleanupPhase(app: Application): Promise<CleanupStats>;
+/**
+ * Get status of all batch jobs
+ */
+export declare function getBatchStatus(app: Application): Promise<BatchStatus[]>;
+/**
+ * Run full batch crawl cycle (submit → process → cleanup)
+ */
+export declare function runBatchCrawlCycle(app: Application): Promise<CycleStats>;
+export { getBatchCrawlerState, BatchCrawlerState };

package/lib/jobs/crawler-state.d.ts

@@ -1,4 +1,4 @@
-import { CrawlStats } from "./crawler";
+import { CrawlStats } from './crawler';
 export type CrawlerState = {
     running: boolean;
     lastRunAt?: string;
@@ -10,3 +10,8 @@ export declare function getCrawlerState(): CrawlerState;
 export declare function startRun(): void;
 export declare function finishRun(result: CrawlStats, durationMs: number): void;
 export declare function failRun(error: unknown, durationMs: number): void;
+/**
+ * Reset state for testing purposes
+ * WARNING: Only use in tests!
+ */
+export declare function resetStateForTesting(): void;

package/lib/jobs/crawler.d.ts

@@ -1,4 +1,4 @@
-import { Application } from "../declarations";
+import { Application } from '../declarations';
 export type CrawlStats = {
     created: number;
     updated: number;

package/lib/jobs/gemini-client-factory.d.ts

@@ -0,0 +1,75 @@
+import { Application } from '../declarations';
+export interface IGeminiClient {
+    models: {
+        generateContent: (params: any) => Promise<any>;
+    };
+    batches: {
+        create: (params: any) => Promise<any>;
+        get: (params: any) => Promise<any>;
+        list: (params?: any) => Promise<any>;
+    };
+    files: {
+        download: (params: any) => Promise<void>;
+    };
+}
+/**
+ * Default configuration applied to all Gemini requests
+ * Includes Google Search grounding tool
+ */
+export declare const GEMINI_DEFAULT_CONFIG: {
+    tools: {
+        googleSearch: {};
+    }[];
+};
+/**
+ * Default Gemini model
+ */
+export declare const DEFAULT_GEMINI_MODEL = "gemini-2.5-flash";
+declare class GeminiClientFactory {
+    private instance;
+    private mockInstance;
+    private model;
+    /**
+     * Initialize the Gemini client from application configuration
+     * Call this once at application startup
+     */
+    initialize(app: Application): void;
+    /**
+     * Get the singleton Gemini client instance
+     * Throws error if not initialized
+     */
+    getClient(): IGeminiClient;
+    /**
+     * Get the configured model name
+     */
+    getModel(): string;
+    /**
+     * Get default configuration with Google Search grounding
+     * Use this in all generateContent calls
+     */
+    getDefaultConfig(): typeof GEMINI_DEFAULT_CONFIG;
+    /**
+     * Set a mock client for testing
+     * WARNING: Only use in tests!
+     */
+    setMockClient(mockClient: IGeminiClient | null): void;
+    /**
+     * Set mock model for testing
+     * WARNING: Only use in tests!
+     */
+    setMockModel(model: string): void;
+    /**
+     * Reset the factory (for testing)
+     * WARNING: Only use in tests!
+     */
+    resetForTesting(): void;
+    /**
+     * Check if client is initialized
+     */
+    isInitialized(): boolean;
+}
+export declare const geminiClientFactory: GeminiClientFactory;
+export declare function getGeminiClient(): IGeminiClient;
+export declare function getGeminiModel(): string;
+export declare function getGeminiConfig(): typeof GEMINI_DEFAULT_CONFIG;
+export {};

package/lib/services/batch-jobs/batch-jobs.class.d.ts

@@ -0,0 +1,15 @@
+import type { Params } from '@feathersjs/feathers';
+import { KnexService } from '@feathersjs/knex';
+import type { KnexAdapterParams, KnexAdapterOptions } from '@feathersjs/knex';
+import type { Application } from '../../declarations';
+import type { BatchJobs, BatchJobsData, BatchJobsPatch, BatchJobsQuery } from './batch-jobs.schema';
+export type { BatchJobs, BatchJobsData, BatchJobsPatch, BatchJobsQuery };
+export interface BatchJobsParams extends KnexAdapterParams<BatchJobsQuery> {
+}
+export declare class BatchJobsService<ServiceParams extends Params = BatchJobsParams> extends KnexService<BatchJobs, BatchJobsData, BatchJobsParams, BatchJobsPatch> {
+    findPending(): Promise<BatchJobs[]>;
+    markCompleted(id: number, resultData: Partial<BatchJobs>): Promise<BatchJobs>;
+    markFailed(id: number, error: string): Promise<BatchJobs>;
+    cleanupOld(daysOld: number): Promise<number>;
+}
+export declare const getOptions: (app: Application) => KnexAdapterOptions;

package/lib/services/batch-jobs/batch-jobs.d.ts

@@ -0,0 +1,11 @@
+import type { Application } from '../../declarations';
+import { BatchJobsService } from './batch-jobs.class';
+import { batchJobsPath } from './batch-jobs.shared';
+export * from './batch-jobs.class';
+export * from './batch-jobs.schema';
+export declare const batchJobs: (app: Application) => void;
+declare module '../../declarations' {
+    interface ServiceTypes {
+        [batchJobsPath]: BatchJobsService;
+    }
+}