hazo_collab_forms 3.0.13 → 3.0.17

package/dist/components/hazo_fb_form/hooks/use_llm_run.js

@@ -44,7 +44,7 @@ function build_tag_group_map(back_sections) {
 }
 /** Route classified files to back-office groups and trigger autofill */
 async function route_files_to_back_office(options) {
-    const { classifications, back_sections, front_form_data, on_back_change, back_form_data, autofill_api_endpoint, file_manager, update_progress, set_group_autofill_log, run_log, tracker, on_autofill_file, force_autofill } = options;
+    const { classifications, back_sections, front_form_data, on_back_change, back_form_data, autofill_api_endpoint, file_manager, update_progress, set_group_autofill_log, run_log, tracker, on_autofill_file, on_queue_files, force_autofill } = options;
     const tag_group_map = build_tag_group_map(back_sections);
     const errors = [];
     const unassigned = [];
@@ -54,7 +54,17 @@ async function route_files_to_back_office(options) {
         const field_files = get_files_from_value(front_form_data[cls.field_id]);
         for (const fc of cls.file_classifications) {
             const file_info = field_files.find((f) => f.file_id === fc.file_id);
-            all_files.push({ ...fc, source_field_id: cls.field_id, attachment: file_info?.attachment });
+            // Build attachment from classification data if not found in front_form_data
+            const attachment = file_info?.attachment ?? {
+                file_id: fc.file_id,
+                ref_id: `ref_${fc.file_id}`,
+                file_name: fc.file_name,
+                file_size: 0,
+                mime_type: 'application/octet-stream',
+                visibility: 'public',
+                attached_at: new Date().toISOString(),
+            };
+            all_files.push({ ...fc, source_field_id: cls.field_id, attachment });
         }
     }
     // Track accumulated files per group locally to avoid stale state reads.
@@ -134,10 +144,17 @@ async function route_files_to_back_office(options) {
         let autofill_done = 0;
         // Count total NEW files across all groups for progress
         let total_autofills = 0;
+        const all_autofill_file_ids = [];
         for (const gid of groups_to_autofill) {
             const fk = `__files_${gid}`;
             const new_files = newly_routed_files.get(fk) ?? [];
             total_autofills += Math.max(new_files.length, 1);
+            for (const f of new_files)
+                all_autofill_file_ids.push(f.file_id);
+        }
+        // Mark all files as queued before autofill starts
+        if (all_autofill_file_ids.length > 0) {
+            on_queue_files?.(all_autofill_file_ids, true);
         }
         // Add autofill actions to the pipeline tracker (now that we know the count)
         if (tracker) {
@@ -180,6 +197,7 @@ async function route_files_to_back_office(options) {
                         current_step: `${pct}Auto-filling ${group_id} — file ${fi + 1}/${group_files.length} (${autofill_done}/${total_autofills})`,
                         overall_percent: tracker ? Math.round((tracker.completed / tracker.total) * 100) : undefined,
                     });
+                    on_queue_files?.([f.file_id], false);
                     on_autofill_file?.(f.file_id, true);
                     try {
                         const download_url = file_manager?.callbacks?.get_download_url?.(f.file_id, 'public') ?? '';
@@ -265,6 +283,8 @@ async function route_files_to_back_office(options) {
                         on_autofill_file?.(f.file_id, false);
                     }
                 }
+                // Collect run_log entries for this group
+                const group_run_log = run_log.filter((entry) => entry.step === 'autofill' && entry.label.startsWith(`Autofill: ${group_id}`));
                 // Set autofill log for this group
                 if (group_error && group_fields_populated === 0) {
                     // If there was an error and no fields were populated, check if it's a real error or just empty
@@ -275,6 +295,7 @@ async function route_files_to_back_office(options) {
                             status: is_empty ? 'empty' : 'error',
                             message: group_error,
                             timestamp: Date.now(),
+                            run_log: group_run_log.length > 0 ? group_run_log : undefined,
                         },
                     }));
                 }
@@ -287,6 +308,7 @@ async function route_files_to_back_office(options) {
                             message: `Populated ${group_fields_populated} field(s) from ${file_label}`,
                             timestamp: Date.now(),
                             details,
+                            run_log: group_run_log.length > 0 ? group_run_log : undefined,
                         },
                     }));
                 }
@@ -297,6 +319,7 @@ async function route_files_to_back_office(options) {
                             status: 'empty',
                             message: 'No matching data found in document',
                             timestamp: Date.now(),
+                            run_log: group_run_log.length > 0 ? group_run_log : undefined,
                         },
                     }));
                 }
@@ -384,9 +407,11 @@ async function validate_classified_files(options) {
         const check_type_filter = options.check_type_filter;
         // check_type filter: rules without check_type default to 'immediate' context.
         // When filtering for 'backoffice', only explicitly backoffice rules run.
+        // check_type filter: rules without check_type run in ALL contexts (immediate + backoffice).
+        // Only rules with an explicit check_type are restricted to that context.
         const matching_rules = validation_rules.filter((r) => r.enabled && (file.document_types.includes(r.document_type) ||
             r.document_type === 'general' ||
-            file.document_types.length === 0) && (!check_type_filter || r.check_type === check_type_filter || (!r.check_type && check_type_filter === 'immediate')));
+            file.document_types.length === 0) && (!check_type_filter || !r.check_type || r.check_type === check_type_filter));
         if (matching_rules.length === 0) {
             // No rules match — preserve existing result if available (e.g. immediate passed),
             // otherwise mark as skipped
@@ -1140,13 +1165,14 @@ export function use_llm_run({ props, update_progress, classification_results, se
                 }
             }
         }
-        // Determine eligible files: passed/skipped validation, OR failed but all clarifications resolved
+        // Determine eligible files: passed/skipped/pending/validating, OR failed but all clarifications resolved
         const eligible_classifications = [];
         for (const cls of classification_results) {
             const eligible_file_cls = [];
             for (const fc of cls.file_classifications) {
                 const vr = file_validation_results[fc.file_id];
-                if (!vr || vr.status === 'passed' || vr.status === 'skipped') {
+                if (!vr || vr.status === 'passed' || vr.status === 'skipped' || vr.status === 'pending' || vr.status === 'validating' || vr.status === 'manual_review') {
+                    // No validation, passed, skipped, manual_review, or validation not yet complete — eligible
                     eligible_file_cls.push(fc);
                 }
                 else if (vr.status === 'failed') {
@@ -1161,8 +1187,17 @@ export function use_llm_run({ props, update_progress, classification_results, se
                     if (all_resolved) {
                         eligible_file_cls.push(fc);
                     }
+                    else {
+                        logger.warn('[use_llm_run] file ineligible: failed validation with unresolved errors', { file_id: fc.file_id, file_name: fc.file_name, error_count: vr.errors.length });
+                    }
+                }
+                else if (vr.status === 'manual_review') {
+                    // Client requested manual review — still eligible for routing/autofill
+                    eligible_file_cls.push(fc);
+                }
+                else {
+                    logger.warn('[use_llm_run] file ineligible: unexpected validation status', { file_id: fc.file_id, status: vr.status });
                 }
-                // manual_review and other statuses: not eligible unless explicitly skipped
             }
             if (eligible_file_cls.length > 0) {
                 eligible_classifications.push({
@@ -1233,6 +1268,12 @@ export function use_llm_run({ props, update_progress, classification_results, se
             set_group_autofill_log,
             run_log,
             on_autofill_file: handle_autofill_file,
+            on_queue_files: (file_ids, queued) => {
+                if (queued)
+                    add_queued_files(file_ids);
+                else
+                    remove_queued_files(file_ids);
+            },
         });
         set_unassigned_files(routing_result.unassigned);
         logger.info('[use_llm_run] trigger_complete_done', {
@@ -1262,39 +1303,86 @@ export function use_llm_run({ props, update_progress, classification_results, se
             classification_count: classification_results.length,
             has_validation: !!(validation_api_endpoint && validation_rules?.length),
             rule_count: validation_rules?.length ?? 0,
+            has_on_back_change: !!on_back_change,
+            has_autofill_endpoint: !!autofill_api_endpoint,
+            back_sections_count: back_sections.length,
+            back_section_groups: back_sections.map(s => (s.groups ?? []).map((g) => ({ id: g.id, tag_id: g.tag_id, prompt_area: g.prompt_area, prompt_key: g.prompt_key }))),
         });
-        if (!on_back_change)
+        if (!on_back_change) {
+            logger.warn('[use_llm_run] trigger_backoffice_run: no on_back_change — aborting');
             return;
-        if (classification_results.length === 0)
+        }
+        if (classification_results.length === 0) {
+            logger.warn('[use_llm_run] trigger_backoffice_run: no classification_results — aborting');
             return;
+        }
         set_group_autofill_log({});
         const run_log = [];
         const errors = [];
+        // ── Queue all classified files as "queued" so UI shows pending state ──
+        const all_classified_file_ids = classification_results.flatMap(c => c.file_classifications.map(fc => fc.file_id));
+        if (all_classified_file_ids.length > 0) {
+            add_queued_files(all_classified_file_ids);
+        }
+        // ── Snapshot refs BEFORE backoffice validation ──
+        // Backoffice validation triggers set_file_validation_results → useEffect clears/replaces
+        // draft_clarifications via React state flush during await. Capture stable copies now.
+        const snapshot_sent = [...sent_clarifications_ref.current];
+        const snapshot_pending = new Map(pending_responses_ref.current);
+        const snapshot_drafts = [...draft_clarifications_ref.current];
+        // Build unified clarification items view from the snapshot
+        const all_clarification_items = new Map();
+        for (const item of snapshot_drafts)
+            all_clarification_items.set(item.id, item);
+        for (const item of snapshot_sent)
+            all_clarification_items.set(item.id, item);
+        for (const [clar_id, response] of snapshot_pending) {
+            const existing = all_clarification_items.get(clar_id);
+            if (existing && response.response_choice) {
+                all_clarification_items.set(clar_id, {
+                    ...existing,
+                    status: 'responded',
+                    response_choice: response.response_choice,
+                    user_comment: response.user_comment,
+                    response_files: response.response_files ?? existing.response_files ?? [],
+                });
+            }
+        }
+        // Build resolved rules map from snapshot (for backoffice validation to skip already-resolved rules)
+        let effective_clarifications_for_rules = snapshot_sent;
+        if (snapshot_pending.size > 0) {
+            const sent_responded_ids = new Set(snapshot_sent.filter(c => c.status === 'responded' || c.status === 'resolved').map(c => c.id));
+            const pending_items = [];
+            for (const [clar_id, response] of snapshot_pending) {
+                if (!response.response_choice)
+                    continue;
+                if (sent_responded_ids.has(clar_id))
+                    continue;
+                const original = all_clarification_items.get(clar_id);
+                if (original) {
+                    pending_items.push({ ...original, status: 'responded', response_choice: response.response_choice, user_comment: response.user_comment });
+                }
+            }
+            if (pending_items.length > 0) {
+                effective_clarifications_for_rules = [...snapshot_sent, ...pending_items];
+            }
+        }
         // ── Step 1: Backoffice validation ──
+        let effective_file_validation = { ...file_validation_results };
         const has_backoffice_validation = !!(validation_api_endpoint && validation_rules?.length);
         if (has_backoffice_validation) {
             update_progress({ status: 'validating', total_steps: 1, completed_steps: 0, current_step: 'Running back-office validation...', error: undefined });
-            const current_sent = sent_clarifications_ref.current;
-            const current_pending = pending_responses_ref.current;
-            const current_drafts = draft_clarifications_ref.current;
-            let effective_clarifications = current_sent;
-            if (current_pending.size > 0) {
-                const pending_items = [];
-                for (const [clar_id, response] of current_pending) {
-                    if (!response.response_choice)
-                        continue;
-                    if (current_sent.some(c => c.id === clar_id && (c.status === 'responded' || c.status === 'resolved')))
-                        continue;
-                    const original = current_drafts.find(c => c.id === clar_id) ?? current_sent.find(c => c.id === clar_id);
-                    if (original) {
-                        pending_items.push({ ...original, status: 'responded', response_choice: response.response_choice, user_comment: response.user_comment });
+            const resolved_rules = build_resolved_rules_map(effective_clarifications_for_rules);
+            // Also mark accepted rules as resolved so they skip re-validation
+            for (const [, result] of Object.entries(effective_file_validation)) {
+                if (result.rule_results) {
+                    for (const rr of result.rule_results) {
+                        if (rr.accepted && rr.rule_id) {
+                            resolved_rules.set(rr.rule_id, { response_choice: rr.user_resolution?.response_choice ?? 'accepted', user_comment: rr.user_resolution?.user_comment });
+                        }
                     }
                 }
-                if (pending_items.length > 0) {
-                    effective_clarifications = [...current_sent, ...pending_items];
-                }
             }
-            const resolved_rules = build_resolved_rules_map(effective_clarifications);
             const vr = await validate_classified_files({
                 classifications: classification_results,
                 validation_api_endpoint: validation_api_endpoint,
@@ -1309,37 +1397,140 @@ export function use_llm_run({ props, update_progress, classification_results, se
                 resolved_rules,
             });
             errors.push(...vr.errors);
+            // Merge backoffice validation results into local copy for eligibility check
+            for (const result of vr.all_validation_results) {
+                effective_file_validation[result.file_id] = result;
+            }
         }
-        // ── Step 2: Determine eligible files and route + autofill ──
-        update_progress({ status: 'routing', total_steps: 1, completed_steps: 0, current_step: 'Routing files to back-office...', error: undefined });
-        const current_sent = sent_clarifications_ref.current;
-        const current_pending = pending_responses_ref.current;
-        const current_drafts = draft_clarifications_ref.current;
-        // Build unified clarification items view
-        const all_clarification_items = new Map();
-        for (const item of current_drafts)
-            all_clarification_items.set(item.id, item);
-        for (const item of current_sent)
-            all_clarification_items.set(item.id, item);
-        for (const [clar_id, response] of current_pending) {
-            const existing = all_clarification_items.get(clar_id);
-            if (existing && response.response_choice) {
-                all_clarification_items.set(clar_id, {
-                    ...existing,
-                    status: 'responded',
-                    response_choice: response.response_choice,
-                    user_comment: response.user_comment,
-                    response_files: response.response_files ?? existing.response_files ?? [],
-                });
+        // ── Step 1b: Text classification ──
+        // Extract text from front-office fields (non-file content) and classify it
+        const { field_textbox_configs, llm_api_endpoint } = props;
+        if (field_textbox_configs && llm_api_endpoint) {
+            const text_entries = [];
+            for (const field_id of Object.keys(field_textbox_configs)) {
+                const text = get_text_from_value(front_form_data[field_id]);
+                if (text.trim().length > 0) {
+                    text_entries.push({ field_id, text });
+                }
+            }
+            if (text_entries.length > 0) {
+                update_progress({ current_step: `Classifying ${text_entries.length} text entry(ies)...` });
+                for (const entry of text_entries) {
+                    try {
+                        const classify_text_start = Date.now();
+                        const response = await fetch(llm_api_endpoint, {
+                            method: 'POST',
+                            headers: { 'Content-Type': 'application/json' },
+                            body: JSON.stringify({
+                                action: 'classify_text',
+                                field_id: entry.field_id,
+                                text: entry.text,
+                                available_tags: field_textbox_configs[entry.field_id]?.classification?.available_tags ?? props.available_tags ?? [],
+                                ...(props.available_document_types?.length ? { available_document_types: props.available_document_types } : {}),
+                            }),
+                        });
+                        const result = await response.json();
+                        run_log.push({
+                            step: 'classify_text',
+                            label: `Text Classification: ${entry.field_id}`,
+                            request: { action: 'classify_text', field_id: entry.field_id, text_length: entry.text.length },
+                            response: result,
+                            timestamp: classify_text_start,
+                            duration_ms: Date.now() - classify_text_start,
+                        });
+                        if (result.success && result.tags?.length > 0) {
+                            // Add text classification to classification_results so review queue can use categories
+                            set_classification_results((prev) => {
+                                const text_cls_id = `__text_${entry.field_id}`;
+                                const text_file_cls = {
+                                    file_id: text_cls_id,
+                                    file_name: `Text from ${entry.field_id}`,
+                                    tags: result.tags,
+                                    document_nature: result.document_nature,
+                                };
+                                const existing = prev.find((c) => c.field_id === entry.field_id);
+                                if (existing) {
+                                    // Merge with existing file classifications
+                                    const filtered = existing.file_classifications.filter((fc) => fc.file_id !== text_cls_id);
+                                    return [
+                                        ...prev.filter((c) => c.field_id !== entry.field_id),
+                                        {
+                                            ...existing,
+                                            tags: [...new Set([...existing.tags, ...result.tags])],
+                                            file_classifications: [...filtered, text_file_cls],
+                                        },
+                                    ];
+                                }
+                                return [...prev, { field_id: entry.field_id, tags: result.tags, file_classifications: [text_file_cls] }];
+                            });
+                        }
+                        // Create draft clarifications from text issues
+                        if (result.success && result.issues?.length > 0) {
+                            const text_clarifications = result.issues.map((issue, idx) => ({
+                                id: `text_issue_${entry.field_id}_${idx}_${Date.now()}`,
+                                type: 'validation_issue',
+                                status: 'pending',
+                                target_field_id: entry.field_id,
+                                target_label: `Text from ${entry.field_id}`,
+                                issue_description: issue.description,
+                                severity: issue.severity === 'error' ? 'error' : 'warning',
+                                doc_references: [{
+                                        file_id: `__text_${entry.field_id}`,
+                                        file_name: `Text from ${entry.field_id}`,
+                                        mime_type: 'text/plain',
+                                    }],
+                                response_options: [
+                                    { value: 'acknowledge', label: 'Acknowledge' },
+                                    { value: 'will_fix', label: 'Will fix' },
+                                    { value: 'ignore', label: 'Ignore' },
+                                ],
+                                created_at: new Date().toISOString(),
+                            }));
+                            set_file_validation_results((prev) => ({
+                                ...prev,
+                                [`__text_${entry.field_id}`]: {
+                                    file_id: `__text_${entry.field_id}`,
+                                    file_name: `Text from ${entry.field_id}`,
+                                    status: 'failed',
+                                    errors: text_clarifications,
+                                    document_types: [],
+                                },
+                            }));
+                        }
+                    }
+                    catch (err) {
+                        errors.push({ step: `classify_text:${entry.field_id}`, error: err instanceof Error ? err.message : 'Text classification failed' });
+                        logger.error('[use_llm_run] classify_text_error', { field_id: entry.field_id, error: err instanceof Error ? err.message : String(err) });
+                    }
+                }
+                logger.info('[use_llm_run] text_classification_phase_complete', { text_entry_count: text_entries.length });
             }
         }
-        // Determine eligible files
+        // Clear queued state — autofill will re-queue eligible files via on_queue_files
+        if (all_classified_file_ids.length > 0) {
+            remove_queued_files(all_classified_file_ids);
+        }
+        // ── Step 2: Determine eligible files and route + autofill ──
+        update_progress({ status: 'routing', total_steps: 1, completed_steps: 0, current_step: 'Routing files to back-office...', error: undefined });
+        // Determine eligible files — use effective_file_validation which merges backoffice results
         const eligible_classifications = [];
         for (const cls of classification_results) {
             const eligible_file_cls = [];
             for (const fc of cls.file_classifications) {
-                const vr = file_validation_results[fc.file_id];
-                if (!vr || vr.status === 'passed' || vr.status === 'skipped') {
+                const vr = effective_file_validation[fc.file_id];
+                // Exclude files with any rule sent back to client (pending client fix)
+                const has_sent_back = vr?.rule_results?.some(rr => rr.sent_back);
+                if (has_sent_back) {
+                    logger.warn('[use_llm_run] backoffice: file ineligible — has sent-back rule pending client response', { file_id: fc.file_id });
+                    continue;
+                }
+                // Exclude files with unreviewed resolved rules (pending agent acceptance)
+                const resolved_rules_for_file = vr?.rule_results?.filter(rr => rr.has_issue && rr.user_resolved) ?? [];
+                if (resolved_rules_for_file.length > 0 && !resolved_rules_for_file.every(rr => rr.accepted)) {
+                    logger.warn('[use_llm_run] backoffice: file ineligible — has unaccepted resolved rules', { file_id: fc.file_id });
+                    continue;
+                }
+                if (!vr || vr.status === 'passed' || vr.status === 'skipped' || vr.status === 'pending' || vr.status === 'validating' || vr.status === 'manual_review') {
                     eligible_file_cls.push(fc);
                 }
                 else if (vr.status === 'failed') {
@@ -1351,6 +1542,11 @@ export function use_llm_run({ props, update_progress, classification_results, se
                     });
                     if (all_resolved)
                         eligible_file_cls.push(fc);
+                    else
+                        logger.warn('[use_llm_run] backoffice: file ineligible — failed with unresolved errors', { file_id: fc.file_id, error_count: vr.errors.length, errors: vr.errors.map(e => ({ id: e.id, status: e.status, rule_id: e.rule_id })) });
+                }
+                else {
+                    logger.warn('[use_llm_run] backoffice: file ineligible — unexpected status', { file_id: fc.file_id, status: vr.status });
                 }
             }
             if (eligible_file_cls.length > 0) {
@@ -1415,6 +1611,12 @@ export function use_llm_run({ props, update_progress, classification_results, se
             set_group_autofill_log,
             run_log,
             on_autofill_file: handle_autofill_file,
+            on_queue_files: (file_ids, queued) => {
+                if (queued)
+                    add_queued_files(file_ids);
+                else
+                    remove_queued_files(file_ids);
+            },
             force_autofill: true,
         });
         set_unassigned_files(routing_result.unassigned);