@vertesia/common 1.5.0-dev.20260725.083715Z → 1.5.0-dev.20260804.124748Z

package/src/api-schemas/indexing.ts

@@ -0,0 +1,234 @@
+import { z } from 'zod';
+import { WorkflowExecutionStatusSchema } from './document-processing.js';
+import { ElasticsearchBackendSchema, ProjectSearchTierSchema } from './project-configuration.js';
+
+const nullableStringSchema: z.ZodType<string | null> = z.any().meta({ type: ['string', 'null'] });
+const nullableNumberSchema: z.ZodType<number | null> = z.any().meta({ type: ['number', 'null'] });
+
+export const StartProjectReindexPayloadSchema = z
+    .strictObject({
+        shard_size: z.number().optional(),
+        parallel_shard_count: z.number().optional(),
+        concurrency: z.number().optional(),
+        bulk_size_bytes: z.number().optional(),
+        bulk_concurrency: z.number().optional(),
+    })
+    .meta({ id: 'StartProjectReindexPayload' });
+
+export const ReindexAgentRunsResponseSchema = z
+    .strictObject({
+        status: z.string(),
+        backend: ElasticsearchBackendSchema,
+        index_name: z.string(),
+        recreated: z.boolean(),
+        total: z.number(),
+        scanned: z.number(),
+        indexed: z.number(),
+        failed: z.number(),
+        errors: z
+            .array(
+                z.strictObject({
+                    id: z.string(),
+                    message: z.string(),
+                }),
+            )
+            .optional(),
+    })
+    .meta({ id: 'ReindexAgentRunsResponse' });
+
+export const ReindexAgentRunsPayloadSchema = z
+    .strictObject({
+        recreate_index: z
+            .boolean()
+            .meta({
+                description:
+                    'Drop any existing agent-runs index/alias family and recreate the stable concrete index before indexing. Defaults to true.',
+            })
+            .optional(),
+        batch_size: z
+            .number()
+            .meta({ description: 'Number of MongoDB records to scan per batch. Defaults to 500.' })
+            .optional(),
+        limit: z
+            .number()
+            .meta({
+                description: 'Optional cap for partial/manual repair runs. Omit for all agent runs in the project.',
+            })
+            .optional(),
+    })
+    .meta({ id: 'ReindexAgentRunsPayload' });
+
+export const IndexingStatusResponseSchema = z
+    .strictObject({
+        infrastructure_enabled: z
+            .boolean()
+            .meta({ description: 'Whether indexing infrastructure is available globally' }),
+        indexing_enabled: z.boolean().meta({ description: 'Whether indexing is enabled for this project' }),
+        query_enabled: z.boolean().meta({
+            description:
+                'Deprecated: Now derived from indexing_enabled - queries automatically route to index when indexing is enabled',
+            deprecated: true,
+            'x-deprecated-message':
+                'Now derived from indexing_enabled - queries automatically route to index when indexing is enabled',
+        }),
+        backend: ElasticsearchBackendSchema.meta({
+            description: 'Resolved Elasticsearch backend serving this project',
+        }),
+        search_tier: ProjectSearchTierSchema.meta({ description: 'Resolved search tier for this project' }),
+        index: z
+            .strictObject({
+                exists: z.boolean().meta({ description: 'Whether the index exists' }),
+                alias_name: z.string().meta({ description: 'Alias name (used for queries)' }),
+                index_name: z.string().meta({ description: 'Actual index name (versioned)' }),
+                version: z.number().meta({ description: 'Index version (timestamp when created)' }),
+                created_at: nullableStringSchema.meta({ description: 'When the current index was created' }),
+                document_count: z.number().meta({ description: 'Number of documents in the index' }),
+                size_bytes: z.number().meta({ description: 'Index size in bytes' }),
+            })
+            .meta({ description: 'Index status' }),
+        mongo_document_count: z.number().meta({ description: 'MongoDB document count for comparison' }),
+        reindex_in_progress: z.boolean().meta({ description: 'Whether a reindex is currently in progress' }),
+        reindex_progress: z
+            .strictObject({
+                total_shards: z.number().meta({ description: 'Total shards to process' }),
+                completed_shards: z.number().meta({ description: 'Shards completed so far' }),
+                failed_shards: z.number().meta({ description: 'Shards that failed' }),
+                status: z
+                    .string()
+                    .meta({ description: 'Current status (e.g., "computing_shards", "indexing", "completed")' }),
+                scanned: z.number().meta({ description: 'Documents scanned from source' }),
+                written: z.number().meta({ description: 'Documents written to target index' }),
+                errors: z.number().meta({ description: 'Documents that failed to index' }),
+                embeddings_written: z
+                    .number()
+                    .meta({ description: 'Embedding vectors written to target index' })
+                    .optional(),
+                skipped_embeddings: z
+                    .number()
+                    .meta({
+                        description: 'Embedding vectors skipped because they were invalid or dimension-mismatched',
+                    })
+                    .optional(),
+                embeddings_text_written: z
+                    .number()
+                    .meta({ description: 'Text embedding vectors written to target index' })
+                    .optional(),
+                embeddings_image_written: z
+                    .number()
+                    .meta({ description: 'Image embedding vectors written to target index' })
+                    .optional(),
+                embeddings_properties_written: z
+                    .number()
+                    .meta({ description: 'Properties embedding vectors written to target index' })
+                    .optional(),
+                embeddings_text_skipped: z
+                    .number()
+                    .meta({
+                        description: 'Text embedding vectors skipped because they were invalid or dimension-mismatched',
+                    })
+                    .optional(),
+                embeddings_image_skipped: z
+                    .number()
+                    .meta({
+                        description:
+                            'Image embedding vectors skipped because they were invalid or dimension-mismatched',
+                    })
+                    .optional(),
+                embeddings_properties_skipped: z
+                    .number()
+                    .meta({
+                        description:
+                            'Properties embedding vectors skipped because they were invalid or dimension-mismatched',
+                    })
+                    .optional(),
+                properties_values_trimmed: z
+                    .number()
+                    .meta({
+                        description: 'Oversized property string values dropped during transform (size-based pruning)',
+                    })
+                    .optional(),
+                properties_bytes_dropped: z
+                    .number()
+                    .meta({ description: 'Total bytes dropped from oversized property values' })
+                    .optional(),
+                batches_flushed: z
+                    .number()
+                    .meta({ description: 'Total batcher flushes across all completed shards (cumulative)' })
+                    .optional(),
+                bulk_chunks_written: z
+                    .number()
+                    .meta({ description: 'Total ES bulk requests sent across all completed shards (cumulative)' })
+                    .optional(),
+                bulk_errors: z
+                    .number()
+                    .meta({
+                        description:
+                            "Total per-document ES bulk-item failures across all shards (cumulative). Counts docs ES rejected — they aren't in the indexed set.",
+                    })
+                    .optional(),
+                avg_docs_per_batch: z
+                    .number()
+                    .meta({
+                        description:
+                            'Average documents per batch flush (written / batches_flushed) — useful to spot under/over-batching',
+                    })
+                    .optional(),
+                avg_chunks_per_batch: z
+                    .number()
+                    .meta({
+                        description:
+                            'Average chunks per batch (>1 means bulk_size_bytes cap is splitting batches frequently)',
+                    })
+                    .optional(),
+                docs_per_second: z.number().meta({ description: 'Documents processed per second' }),
+                elapsed_seconds: z.number().meta({ description: 'Elapsed time in seconds' }),
+                estimated_seconds_remaining: nullableNumberSchema.meta({
+                    description: 'Estimated seconds remaining (null if unknown)',
+                }),
+                percent_complete: z.number().meta({ description: 'Percentage complete (0-100)' }),
+                alias: z.string().meta({ description: 'Source alias' }),
+                target_index: z.string().meta({ description: 'Target index name' }),
+            })
+            .meta({ description: 'Reindex progress (if reindex is in progress)' })
+            .optional(),
+    })
+    .meta({ id: 'IndexingStatusResponse', description: 'Response from indexing status endpoint' });
+
+export const DriftAnalysisResultSchema = z
+    .strictObject({
+        total: z.number(),
+        processed: z.number(),
+        missing: z.number(),
+        stale: z.number(),
+        sample_missing_ids: z.array(z.string()),
+        sample_stale_ids: z.array(z.string()),
+        completed_at: z.string(),
+    })
+    .meta({ id: 'DriftAnalysisResult' });
+
+export const DriftAnalysisProgressSchema = z
+    .strictObject({
+        total: z.number(),
+        processed: z.number(),
+        missing: z.number(),
+        stale: z.number(),
+        status: z.string(),
+        current_batch: z.number(),
+        total_batches: z.number(),
+        percent_complete: z.number(),
+        docs_per_second: z.number(),
+        elapsed_seconds: z.number(),
+        estimated_seconds_remaining: nullableNumberSchema,
+    })
+    .meta({ id: 'DriftAnalysisProgress' });
+
+export const DriftAnalysisStatusResponseSchema = z
+    .strictObject({
+        workflow_id: nullableStringSchema,
+        workflow_run_id: nullableStringSchema,
+        status: WorkflowExecutionStatusSchema,
+        progress: DriftAnalysisProgressSchema.optional(),
+        result: DriftAnalysisResultSchema.optional(),
+        error: z.string().optional(),
+    })
+    .meta({ id: 'DriftAnalysisStatusResponse' });

package/src/api-schemas/integrations.ts

@@ -0,0 +1,369 @@
+// Runtime schemas for the integrations API domain.
+
+import { z } from 'zod';
+import { SupportedIntegrations } from '../integrations.js';
+import { StringValueMapSchema } from './files.js';
+import { DSLActivityOptionsSchema } from './process.js';
+
+export const SupportedIntegrations_ask_user_webhookSchema = z
+    .literal(SupportedIntegrations.ask_user_webhook)
+    .meta({ id: 'SupportedIntegrations_ask_user_webhook' });
+
+export const SupportedIntegrations_resendSchema = z
+    .literal(SupportedIntegrations.resend)
+    .meta({ id: 'SupportedIntegrations_resend' });
+
+export const SupportedIntegrations_linkupSchema = z
+    .literal(SupportedIntegrations.linkup)
+    .meta({ id: 'SupportedIntegrations_linkup' });
+
+export const SupportedIntegrations_exaSchema = z
+    .literal(SupportedIntegrations.exa)
+    .meta({ id: 'SupportedIntegrations_exa' });
+
+export const SupportedIntegrations_serperSchema = z
+    .literal(SupportedIntegrations.serper)
+    .meta({ id: 'SupportedIntegrations_serper' });
+
+export const SupportedIntegrations_magic_pdfSchema = z
+    .literal(SupportedIntegrations.magic_pdf)
+    .meta({ id: 'SupportedIntegrations_magic_pdf' });
+
+export const SupportedIntegrations_awsSchema = z
+    .literal(SupportedIntegrations.aws)
+    .meta({ id: 'SupportedIntegrations_aws' });
+
+export const SupportedIntegrations_githubSchema = z
+    .literal(SupportedIntegrations.github)
+    .meta({ id: 'SupportedIntegrations_github' });
+
+export const SupportedIntegrations_gladiaSchema = z
+    .literal(SupportedIntegrations.gladia)
+    .meta({ id: 'SupportedIntegrations_gladia' });
+
+export const AskUserWebhookConfigurationSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_ask_user_webhookSchema,
+        enabled: z.boolean(),
+        webhook_url: z.string().meta({ description: 'Webhook URL to receive ask_user events' }),
+        has_webhook_secret: z.boolean().optional(),
+        webhook_secret_hint: z.string().optional(),
+        events: z
+            .array(z.enum(['requested', 'resolved']))
+            .meta({ description: "Which events to send: ['requested', 'resolved'] or subset (default: both)" })
+            .optional(),
+        custom_headers: StringValueMapSchema.meta({
+            description: 'Custom headers to include in webhook requests',
+        }).optional(),
+    })
+    .meta({
+        id: 'AskUserWebhookConfiguration',
+        description:
+            'Configuration for ask_user webhook notifications. Sends webhooks when agents call ask_user and when users respond.',
+    });
+
+export const ResendConfigurationSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_resendSchema,
+        enabled: z.boolean(),
+        has_api_key: z.boolean().optional(),
+        api_key_hint: z.string().optional(),
+        email_domain: z
+            .string()
+            .meta({ description: 'Domain for email (both sending and receiving). Must be verified in Resend.' }),
+        default_from_name: z
+            .string()
+            .meta({ description: 'Default display name for outgoing emails (e.g., "Vertesia - Project Name")' })
+            .optional(),
+        has_webhook_secret: z.boolean().optional(),
+        webhook_secret_hint: z.string().optional(),
+        allowed_sender_domains: z
+            .array(z.string())
+            .meta({ description: 'Domains allowed to send emails TO start agents (for inbound validation)' })
+            .optional(),
+        require_project_access: z
+            .boolean()
+            .meta({ description: 'Require sender to have project access to start agents via email (default: true)' })
+            .optional(),
+        require_email_auth: z
+            .boolean()
+            .meta({ description: 'Require DKIM/SPF authentication to pass for inbound emails (default: true)' })
+            .optional(),
+    })
+    .meta({ id: 'ResendConfiguration' });
+
+export const LinkupConfigurationSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_linkupSchema,
+        enabled: z.boolean(),
+        has_api_key: z.boolean().optional(),
+        api_key_hint: z.string().optional(),
+    })
+    .meta({ id: 'LinkupConfiguration' });
+
+export const ExaConfigurationSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_exaSchema,
+        enabled: z.boolean(),
+        has_api_key: z.boolean().optional(),
+        api_key_hint: z.string().optional(),
+    })
+    .meta({ id: 'ExaConfiguration' });
+
+export const SerperConfigurationSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_serperSchema,
+        enabled: z.boolean(),
+        has_api_key: z.boolean().optional(),
+        api_key_hint: z.string().optional(),
+        url: z.string().optional(),
+    })
+    .meta({ id: 'SerperConfiguration' });
+
+export const GithubConfigurationSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_githubSchema,
+        enabled: z.boolean(),
+        github_app_id: z
+            .string()
+            .meta({ description: 'Numeric GitHub App id used to mint installation tokens (non-secret).' })
+            .optional(),
+        allowed_repositories: z.array(z.string()),
+        has_github_app_private_key: z
+            .boolean()
+            .meta({
+                description:
+                    'True when a GitHub App private key is stored for the project (the key itself is never returned).',
+            })
+            .optional(),
+    })
+    .meta({ id: 'GithubConfiguration' });
+
+export const GladiaConfigurationSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_gladiaSchema,
+        enabled: z.boolean(),
+        has_api_key: z.boolean().optional(),
+        api_key_hint: z.string().optional(),
+        url: z.string().optional(),
+    })
+    .meta({ id: 'GladiaConfiguration' });
+
+export const RemoteActivityDefinitionSchema = z
+    .strictObject({
+        name: z.string().meta({ description: 'Activity name (snake_case, unique within the collection)' }),
+        collection: z.string().meta({ description: 'Collection name this activity belongs to' }).optional(),
+        title: z.string().meta({ description: 'Display title' }).optional(),
+        description: z.string().meta({ description: 'Description of what the activity does' }).optional(),
+        input_schema: z
+            .looseObject({})
+            .meta({ description: 'JSON Schema for the activity input parameters' })
+            .optional(),
+        output_schema: z.looseObject({}).meta({ description: 'JSON Schema for the activity output' }).optional(),
+        url: z
+            .string()
+            .meta({
+                description:
+                    'The activity execution URL. Can be absolute or relative to the tool server base URL. If not provided, the collection-specific activities endpoint is used.',
+            })
+            .optional(),
+        options: DSLActivityOptionsSchema.meta({ description: 'Suggested timeout and retry configuration' }).optional(),
+    })
+    .meta({
+        id: 'RemoteActivityDefinition',
+        description:
+            'Definition of a remote activity exposed by a tool server for use in DSL workflows. Remote activities are identified in workflow steps using colon-separated names: `app:<app_name>:<collection>:<activity_name>` (e.g. `app:my-nlp-app:examples:word_count`).',
+    });
+
+export const AskUserWebhookConfigurationInputSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_ask_user_webhookSchema,
+        enabled: z.boolean(),
+        webhook_url: z.string().meta({ description: 'Webhook URL to receive ask_user events' }),
+        webhook_secret: z
+            .string()
+            .meta({ description: 'Secret for signing webhook payloads (HMAC-SHA256)' })
+            .optional(),
+        events: z
+            .array(z.enum(['requested', 'resolved']))
+            .meta({ description: "Which events to send: ['requested', 'resolved'] or subset (default: both)" })
+            .optional(),
+        custom_headers: StringValueMapSchema.meta({
+            description: 'Custom headers to include in webhook requests',
+        }).optional(),
+    })
+    .meta({
+        id: 'AskUserWebhookConfigurationInput',
+        description:
+            'Configuration for ask_user webhook notifications. Sends webhooks when agents call ask_user and when users respond.',
+    });
+
+export const ResendConfigurationInputSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_resendSchema,
+        enabled: z.boolean(),
+        api_key: z.string().meta({ description: 'Resend API key for sending emails' }).optional(),
+        email_domain: z
+            .string()
+            .meta({ description: 'Domain for email (both sending and receiving). Must be verified in Resend.' }),
+        default_from_name: z
+            .string()
+            .meta({ description: 'Default display name for outgoing emails (e.g., "Vertesia - Project Name")' })
+            .optional(),
+        webhook_secret: z
+            .string()
+            .meta({
+                description: 'Webhook secret for validating inbound email webhooks (required for receiving emails)',
+            })
+            .optional(),
+        allowed_sender_domains: z
+            .array(z.string())
+            .meta({ description: 'Domains allowed to send emails TO start agents (for inbound validation)' })
+            .optional(),
+        require_project_access: z
+            .boolean()
+            .meta({ description: 'Require sender to have project access to start agents via email (default: true)' })
+            .optional(),
+        require_email_auth: z
+            .boolean()
+            .meta({ description: 'Require DKIM/SPF authentication to pass for inbound emails (default: true)' })
+            .optional(),
+    })
+    .meta({ id: 'ResendConfigurationInput' });
+
+export const LinkupConfigurationInputSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_linkupSchema,
+        enabled: z.boolean(),
+        api_key: z.string().optional(),
+    })
+    .meta({ id: 'LinkupConfigurationInput' });
+
+export const ExaConfigurationInputSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_exaSchema,
+        enabled: z.boolean(),
+        api_key: z.string().optional(),
+    })
+    .meta({ id: 'ExaConfigurationInput' });
+
+export const SerperConfigurationInputSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_serperSchema,
+        enabled: z.boolean(),
+        api_key: z.string().optional(),
+        url: z.string().optional(),
+    })
+    .meta({ id: 'SerperConfigurationInput' });
+
+export const MagicPdfConfigurationSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_magic_pdfSchema,
+        enabled: z.boolean(),
+        default_features: z.array(z.string()).optional(),
+        default_zones: z.array(z.string()).optional(),
+    })
+    .meta({ id: 'MagicPdfConfiguration' });
+
+export const AwsConfigurationSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_awsSchema,
+        enabled: z.boolean(),
+        s3_role_arn: z.string(),
+    })
+    .meta({ id: 'AwsConfiguration' });
+
+export const GithubConfigurationInputSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_githubSchema,
+        enabled: z.boolean(),
+        github_app_id: z
+            .string()
+            .meta({ description: 'Numeric GitHub App id used to mint installation tokens (non-secret).' })
+            .optional(),
+        allowed_repositories: z
+            .array(z.string())
+            .meta({
+                description:
+                    'Allow-list of `owner/name` repos the bot may mint installation tokens for (default-deny when empty).',
+            })
+            .optional(),
+        private_key: z
+            .string()
+            .meta({ description: 'GitHub App private key (PEM). Write-only; never returned. Empty string clears it.' })
+            .optional(),
+    })
+    .meta({ id: 'GithubConfigurationInput' });
+
+export const GladiaConfigurationInputSchema = z
+    .strictObject({
+        integration: SupportedIntegrations_gladiaSchema,
+        enabled: z.boolean(),
+        api_key: z.string().optional(),
+        url: z.string().optional(),
+    })
+    .meta({ id: 'GladiaConfigurationInput' });
+
+export const ProjectIntegrationConfigResponseSchema = z
+    .discriminatedUnion('integration', [
+        GladiaConfigurationSchema,
+        GithubConfigurationSchema,
+        AwsConfigurationSchema,
+        MagicPdfConfigurationSchema,
+        SerperConfigurationSchema,
+        ExaConfigurationSchema,
+        LinkupConfigurationSchema,
+        ResendConfigurationSchema,
+        AskUserWebhookConfigurationSchema,
+    ])
+    .meta({
+        id: 'ProjectIntegrationConfigResponse',
+        type: 'object',
+        required: ['integration'],
+        discriminator: {
+            propertyName: 'integration',
+            mapping: {
+                ask_user_webhook: '#/components/schemas/AskUserWebhookConfiguration',
+                aws: '#/components/schemas/AwsConfiguration',
+                exa: '#/components/schemas/ExaConfiguration',
+                github: '#/components/schemas/GithubConfiguration',
+                gladia: '#/components/schemas/GladiaConfiguration',
+                linkup: '#/components/schemas/LinkupConfiguration',
+                magic_pdf: '#/components/schemas/MagicPdfConfiguration',
+                resend: '#/components/schemas/ResendConfiguration',
+                serper: '#/components/schemas/SerperConfiguration',
+            },
+        },
+    });
+
+export const ProjectIntegrationConfigRequestSchema = z
+    .discriminatedUnion('integration', [
+        GladiaConfigurationInputSchema,
+        GithubConfigurationInputSchema,
+        AwsConfigurationSchema,
+        MagicPdfConfigurationSchema,
+        SerperConfigurationInputSchema,
+        ExaConfigurationInputSchema,
+        LinkupConfigurationInputSchema,
+        ResendConfigurationInputSchema,
+        AskUserWebhookConfigurationInputSchema,
+    ])
+    .meta({
+        id: 'ProjectIntegrationConfigRequest',
+        type: 'object',
+        required: ['integration'],
+        discriminator: {
+            propertyName: 'integration',
+            mapping: {
+                ask_user_webhook: '#/components/schemas/AskUserWebhookConfigurationInput',
+                aws: '#/components/schemas/AwsConfiguration',
+                exa: '#/components/schemas/ExaConfigurationInput',
+                github: '#/components/schemas/GithubConfigurationInput',
+                gladia: '#/components/schemas/GladiaConfigurationInput',
+                linkup: '#/components/schemas/LinkupConfigurationInput',
+                magic_pdf: '#/components/schemas/MagicPdfConfiguration',
+                resend: '#/components/schemas/ResendConfigurationInput',
+                serper: '#/components/schemas/SerperConfigurationInput',
+            },
+        },
+    });