@vertesia/common 1.5.0-dev.20260714.072725Z → 1.5.0-dev.20260722.120446Z

package/src/project.ts

@@ -1,7 +1,9 @@
 import type { JSONSchemaType } from 'ajv';
 import type { SupportedIntegrations } from './integrations.js';
+import type { ContentTypeIntakePolicy, IntakeVisionDetail, IntakeVisionProfileSettings } from './store/store.js';
 import type { WorkflowRunStatus } from './store/workflow.js';
 import type { AccountRef } from './user.js';
+import { ELASTICSEARCH_FIELD_PATH_PATTERN } from './view-validation-helpers.js';
 
 export interface ICreateProjectPayload {
     name: string;
@@ -125,6 +127,7 @@ export const SYSTEM_INTERACTION_CATEGORIES: Record<string, SystemInteractionCate
     Mediator: SystemInteractionCategory.non_applicable,
     AnalyzeConversation: SystemInteractionCategory.analysis,
     GetAgentConversationTopic: SystemInteractionCategory.analysis,
+    ContentSearchAgent: SystemInteractionCategory.analysis,
     StudioAssistant: SystemInteractionCategory.agent,
 };
 
@@ -269,7 +272,70 @@ export const BrowserUseProjectConfigurationSchema: JSONSchemaType<BrowserUseProj
 export type ProjectSearchTier = 'standard' | 'performance';
 export type ElasticsearchBackend = 'serverless' | 'hosted';
 
+/**
+ * Fast pre-conversion type identification (the "sniff") for untyped documents.
+ * The sniff classifies a document from cheap local evidence (first/last page text,
+ * a low-res first-page image, office docProps) BEFORE any conversion, so a
+ * high-confidence match can apply the type's intake policy — including skipping
+ * conversion — without paying for it first.
+ */
+export interface ProjectIntakeSniffConfiguration {
+    /**
+     * Enable the pre-conversion sniff for untyped documents. Defaults to true.
+     * Can be overridden per run with the `sniffEnabled` workflow var.
+     */
+    enabled?: boolean;
+
+    /**
+     * Confidence at or above which the sniffed type is committed and its full policy applied
+     * (including conversion-skip). 0..1, defaults to 0.85.
+     */
+    high_confidence?: number;
+
+    /**
+     * Confidence at or above which the sniffed type is treated as provisional: the document
+     * still converts and the post-conversion selector confirms on neutral evidence.
+     * 0..1, defaults to 0.6. Below this the sniff result is advisory provenance only.
+     */
+    medium_confidence?: number;
+
+    /**
+     * Minimum page count for the sniff LLM call. Below this, conversion is cheap and full
+     * converted text is better selection evidence, so intake uses the standard
+     * convert-then-select path. Documents with unknown page counts are sniffed.
+     * Defaults to 5; 0 means always sniff.
+     */
+    min_pages?: number;
+}
+
 export interface ProjectIntakeConfiguration {
+    /**
+     * Master switch for the standard intake pipeline. When false, StandardIntake exits as a
+     * no-op WITHOUT touching object status (objects stay in `created`, identifiable as
+     * unprocessed). Defaults to true.
+     */
+    enabled?: boolean;
+
+    /**
+     * Fast pre-conversion type identification for untyped documents. Absent means enabled
+     * with platform default thresholds.
+     */
+    sniff?: ProjectIntakeSniffConfiguration;
+
+    /**
+     * Project-level intake policy defaults. Same shape as the per-content-type policy; a
+     * type's `intake` block wins field-by-field over these defaults, which in turn win over
+     * the legacy flat fields below. `identification` is type-specific and ignored here.
+     */
+    default_policy?: ContentTypeIntakePolicy;
+
+    /**
+     * Project overrides for the platform vision detail profiles used by intake visual
+     * extraction (`low`/`standard`/`high`). Partial: omitted profiles or fields inherit the
+     * platform defaults. Types reference detail NAMES only; the profile settings live here.
+     */
+    vision_profiles?: Partial<Record<IntakeVisionDetail, Partial<IntakeVisionProfileSettings>>>;
+
     /**
      * Generate table-of-content sections during standard document intake.
      * Defaults to false.
@@ -331,28 +397,7 @@ export interface ProjectConfiguration {
      * Indexing configuration for this project.
      * Controls whether indexing and querying are enabled at the project level.
      */
-    indexing?: {
-        /**
-         * Enable indexing for content objects in this project.
-         * When enabled, content changes trigger indexing workflows.
-         * Defaults to true - indexing is always on when ES infrastructure is available.
-         */
-        enabled?: boolean;
-
-        /**
-         * Search tier for this project.
-         * standard uses the regional hosted Elasticsearch deployment.
-         * performance uses the regional serverless Elasticsearch project.
-         * Defaults to standard when omitted.
-         */
-        search_tier?: ProjectSearchTier;
-
-        /**
-         * Elasticsearch backend override for this project.
-         * Prefer search_tier for project configuration unless an explicit backend override is needed.
-         */
-        backend?: ElasticsearchBackend;
-    };
+    indexing?: ProjectIndexingConfiguration;
 
     /**
      * Standard content intake behavior.
@@ -382,6 +427,142 @@ export interface ProjectConfiguration {
     pdf_template_object_id?: string;
 }
 
+/**
+ * Elasticsearch field types that may be explicitly assigned to content-object
+ * properties. Paths are relative to the object's `properties` field.
+ */
+export type ProjectSearchPropertyType = 'keyword' | 'text' | 'boolean' | 'long' | 'double' | 'date';
+
+/**
+ * Explicit search mapping for one content-object property.
+ *
+ * Changing a mapping requires a full reindex. Existing Elasticsearch fields
+ * cannot change type in place.
+ */
+export interface ProjectSearchPropertyMapping {
+    type: ProjectSearchPropertyType;
+
+    /** Elasticsearch date format. Valid only when type is `date`. */
+    format?: string;
+
+    /** Maximum indexed string length. Valid only when type is `keyword`. */
+    ignore_above?: number;
+
+    /**
+     * Skip malformed values instead of rejecting the whole document. Valid only
+     * for long, double, and date mappings.
+     */
+    ignore_malformed?: boolean;
+}
+
+export const PROJECT_SEARCH_PROPERTY_TYPES: readonly ProjectSearchPropertyType[] = [
+    'keyword',
+    'text',
+    'boolean',
+    'long',
+    'double',
+    'date',
+];
+
+const MAX_PROJECT_SEARCH_PROPERTY_MAPPINGS = 200;
+const MAX_KEYWORD_IGNORE_ABOVE = 8191;
+
+/**
+ * Validate property mappings at API and index-creation boundaries.
+ *
+ * Returns user-facing issue strings instead of throwing so callers can map the
+ * result to the error type appropriate for their boundary.
+ */
+export function validateProjectSearchPropertyMappings(value: unknown): string[] {
+    if (value === undefined) return [];
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        return ['indexing.property_mappings must be an object keyed by property path'];
+    }
+
+    const entries = Object.entries(value as Record<string, unknown>);
+    const issues: string[] = [];
+    if (entries.length > MAX_PROJECT_SEARCH_PROPERTY_MAPPINGS) {
+        issues.push(`indexing.property_mappings must contain at most ${MAX_PROJECT_SEARCH_PROPERTY_MAPPINGS} fields`);
+    }
+
+    const supportedTypes = new Set<string>(PROJECT_SEARCH_PROPERTY_TYPES);
+    for (const [path, rawMapping] of entries) {
+        const field = `indexing.property_mappings.${path}`;
+        if (!ELASTICSEARCH_FIELD_PATH_PATTERN.test(path)) {
+            issues.push(`${field} must be a dot-separated path containing only letters, numbers, and underscores`);
+        }
+        if (!rawMapping || typeof rawMapping !== 'object' || Array.isArray(rawMapping)) {
+            issues.push(`${field} must be an object`);
+            continue;
+        }
+        const mapping = rawMapping as Record<string, unknown>;
+        const extraKeys = Object.keys(mapping).filter(
+            (key) => !['type', 'format', 'ignore_above', 'ignore_malformed'].includes(key),
+        );
+        if (extraKeys.length > 0) {
+            issues.push(`${field} contains unsupported option(s): ${extraKeys.join(', ')}`);
+        }
+        if (typeof mapping.type !== 'string' || !supportedTypes.has(mapping.type)) {
+            issues.push(`${field}.type must be one of: ${PROJECT_SEARCH_PROPERTY_TYPES.join(', ')}`);
+        }
+        if (mapping.format !== undefined && (mapping.type !== 'date' || typeof mapping.format !== 'string')) {
+            issues.push(`${field}.format is supported only for date mappings`);
+        }
+        if (
+            mapping.ignore_above !== undefined &&
+            (mapping.type !== 'keyword' ||
+                !Number.isInteger(mapping.ignore_above) ||
+                (mapping.ignore_above as number) < 1 ||
+                (mapping.ignore_above as number) > MAX_KEYWORD_IGNORE_ABOVE)
+        ) {
+            issues.push(
+                `${field}.ignore_above is supported only for keyword mappings and must be an integer from 1 to ${MAX_KEYWORD_IGNORE_ABOVE}`,
+            );
+        }
+        if (
+            mapping.ignore_malformed !== undefined &&
+            (!['long', 'double', 'date'].includes(String(mapping.type)) ||
+                typeof mapping.ignore_malformed !== 'boolean')
+        ) {
+            issues.push(`${field}.ignore_malformed is supported only for long, double, and date mappings`);
+        }
+    }
+    return issues;
+}
+
+export interface ProjectIndexingConfiguration {
+    /**
+     * Enable indexing for content objects in this project.
+     * When enabled, content changes trigger indexing workflows.
+     * Defaults to true - indexing is always on when ES infrastructure is available.
+     */
+    enabled?: boolean;
+
+    /**
+     * Search tier for this project.
+     * standard uses the regional hosted Elasticsearch deployment.
+     * performance uses the regional serverless Elasticsearch project.
+     * Defaults to standard when omitted.
+     */
+    search_tier?: ProjectSearchTier;
+
+    /**
+     * Elasticsearch backend override for this project.
+     * Prefer search_tier for project configuration unless an explicit backend override is needed.
+     */
+    backend?: ElasticsearchBackend;
+
+    /**
+     * Explicit mappings for selected content-object property paths.
+     *
+     * Keys are dot-separated paths relative to `properties`, for example
+     * `order_total` or `customer.account_number`. Unlisted fields are mapped
+     * dynamically from their JSON values. Changing this value requires a full
+     * reindex.
+     */
+    property_mappings?: Record<string, ProjectSearchPropertyMapping>;
+}
+
 // export interface ProjectConfigurationEmbeddings {
 //     environment: string;
 //     max_tokens: number;
@@ -937,6 +1118,8 @@ export interface IndexConfiguration {
     };
     /** ISO 639-1 language code for text analysis */
     language?: string;
+    /** Explicit mappings for selected content-object property paths. */
+    property_mappings?: Record<string, ProjectSearchPropertyMapping>;
     field_mappings?: Record<string, unknown>;
     project_embeddings_config?: {
         text?: EmbeddingTypeConfig;

package/src/query.ts

@@ -88,6 +88,12 @@ export interface RunSearchQuery extends SimpleSearchQuery {
     model?: string;
     status?: ExecutionRunStatus;
     tags?: string[];
+    /**
+     * Tags to exclude. Runs carrying any of these tags are filtered out of the results,
+     * counts, and facet buckets. Combined with `tags` (which requires all of the listed
+     * tags) as an additional `$nin` constraint on the same field.
+     */
+    exclude_tags?: string[];
     query?: string;
     default_query_path?: string;
     parent?: string[];

package/src/refs.ts

@@ -21,6 +21,7 @@ export interface ResourceRef {
     id: string;
     name: string;
     type: string;
+    email?: string;
     description?: string;
     version?: number;
     status?: string;

package/src/roles.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, test } from 'vitest';
+import { SystemRoles } from './project.js';
+import { AbacScopes } from './roles/types.js';
+
+describe('shared role vocabulary', () => {
+    test('exports all system roles', () => {
+        expect(Object.values(SystemRoles).sort()).toEqual(
+            [
+                'admin',
+                'app_member',
+                'application',
+                'auditor',
+                'automation',
+                'billing',
+                'consumer',
+                'content_processor',
+                'content_superadmin',
+                'developer',
+                'executor',
+                'manager',
+                'member',
+                'owner',
+                'reader',
+                'support',
+            ].sort(),
+        );
+    });
+
+    test('exports ABAC scopes used by role wire types', () => {
+        expect(AbacScopes).toEqual(['document', 'collection', 'task']);
+    });
+});

package/src/schema-for-extraction.test.ts

@@ -0,0 +1,191 @@
+import { describe, expect, it } from 'vitest';
+import {
+    isExtractableSchemaNode,
+    mergePreservingNonExtractable,
+    schemaForExtraction,
+} from './schema-for-extraction.js';
+
+describe('schemaForExtraction', () => {
+    it('drops top-level properties marked x-extract: false and cleans required', () => {
+        const schema = {
+            type: 'object',
+            properties: {
+                po_number: { type: 'string' },
+                match: { type: 'string', 'x-extract': false },
+                match_reason: { type: 'string', 'x-extract': false },
+            },
+            required: ['po_number', 'match'],
+        };
+        const filtered = schemaForExtraction(schema);
+        expect(Object.keys(filtered.properties as object).sort()).toEqual(['po_number']);
+        expect(filtered.required).toEqual(['po_number']);
+        expect((filtered.properties as Record<string, unknown>).po_number).toEqual({ type: 'string' });
+    });
+
+    it('drops nested and array-item properties with x-extract: false', () => {
+        const schema = {
+            type: 'object',
+            properties: {
+                line_items: {
+                    type: 'array',
+                    items: {
+                        type: 'object',
+                        properties: {
+                            part_number: { type: 'string' },
+                            match: { type: 'string', 'x-extract': false },
+                            quantity: { type: 'string' },
+                        },
+                        required: ['part_number', 'match'],
+                    },
+                },
+            },
+        };
+        const filtered = schemaForExtraction(schema);
+        const items = (filtered.properties as Record<string, Record<string, unknown>>).line_items.items as Record<
+            string,
+            unknown
+        >;
+        expect(Object.keys(items.properties as object).sort()).toEqual(['part_number', 'quantity']);
+        expect(items.required).toEqual(['part_number']);
+    });
+
+    it('treats missing x-extract as extractable', () => {
+        expect(isExtractableSchemaNode({ type: 'string' })).toBe(true);
+        expect(isExtractableSchemaNode({ type: 'string', 'x-extract': true })).toBe(true);
+        expect(isExtractableSchemaNode({ type: 'string', 'x-extract': false })).toBe(false);
+    });
+
+    it('filters $defs and drops properties that reference non-extractable definitions', () => {
+        const schema = {
+            type: 'object',
+            $defs: {
+                hidden: { type: 'string', 'x-extract': false },
+                visible: {
+                    type: 'object',
+                    properties: {
+                        value: { type: 'string' },
+                        match: { type: 'string', 'x-extract': false },
+                    },
+                    required: ['value', 'match'],
+                },
+            },
+            properties: {
+                hidden: { $ref: '#/$defs/hidden' },
+                visible: { $ref: '#/$defs/visible' },
+            },
+            required: ['hidden', 'visible'],
+        };
+
+        const filtered = schemaForExtraction(schema);
+
+        expect(filtered.$defs.hidden).toBeUndefined();
+        expect(filtered.$defs.visible).toEqual({
+            type: 'object',
+            properties: { value: { type: 'string' } },
+            required: ['value'],
+        });
+        expect(filtered.properties).toEqual({ visible: { $ref: '#/$defs/visible' } });
+        expect(filtered.required).toEqual(['visible']);
+    });
+
+    it('drops non-extractable anyOf branches', () => {
+        const schema = {
+            type: 'object',
+            properties: {
+                value: {
+                    anyOf: [{ type: 'string', 'x-extract': false }, { type: 'number' }],
+                },
+            },
+        };
+
+        const filtered = schemaForExtraction(schema);
+
+        expect((filtered.properties.value as { anyOf: unknown[] }).anyOf).toEqual([{ type: 'number' }]);
+    });
+});
+
+describe('mergePreservingNonExtractable', () => {
+    it('preserves non-extractable top-level values over model output', () => {
+        const schema = {
+            type: 'object',
+            properties: {
+                po_number: { type: 'string' },
+                match: { type: 'string', 'x-extract': false },
+            },
+        };
+        const existing = { po_number: 'OLD', match: 'erp-hit' };
+        const extracted = { po_number: 'NEW', match: 'model-invented' };
+        expect(mergePreservingNonExtractable(existing, extracted, schema)).toEqual({
+            po_number: 'NEW',
+            match: 'erp-hit',
+        });
+    });
+
+    it('clears extractable fields omitted by the latest extraction', () => {
+        const schema = {
+            type: 'object',
+            properties: {
+                po_number: { type: 'string' },
+                total: { type: 'number' },
+                match: { type: 'string', 'x-extract': false },
+            },
+        };
+        const existing = { po_number: 'STALE', total: 42, match: 'erp-hit' };
+        const extracted = { total: 50 };
+
+        expect(mergePreservingNonExtractable(existing, extracted, schema)).toEqual({
+            total: 50,
+            match: 'erp-hit',
+        });
+    });
+
+    it('does not preserve non-extractable array item fields by index', () => {
+        const schema = {
+            type: 'object',
+            properties: {
+                line_items: {
+                    type: 'array',
+                    items: {
+                        type: 'object',
+                        properties: {
+                            part: { type: 'string' },
+                            match: { type: 'string', 'x-extract': false },
+                        },
+                    },
+                },
+            },
+        };
+        const existing = {
+            line_items: [
+                { part: 'A', match: 'hit-a' },
+                { part: 'B', match: 'hit-b' },
+            ],
+        };
+        const extracted = {
+            line_items: [{ part: 'B2' }],
+        };
+        expect(mergePreservingNonExtractable(existing, extracted, schema)).toEqual({
+            line_items: [{ part: 'B2' }],
+        });
+    });
+
+    it('preserves top-level values referenced through non-extractable $defs', () => {
+        const schema = {
+            type: 'object',
+            $defs: {
+                match: { type: 'string', 'x-extract': false },
+            },
+            properties: {
+                po_number: { type: 'string' },
+                match: { $ref: '#/$defs/match' },
+            },
+        };
+        const existing = { po_number: 'OLD', match: 'erp-hit' };
+        const extracted = { po_number: 'NEW', match: 'model-invented' };
+
+        expect(mergePreservingNonExtractable(existing, extracted, schema)).toEqual({
+            po_number: 'NEW',
+            match: 'erp-hit',
+        });
+    });
+});