@reftrixmcp/mcp-server 0.1.7

package/dist/workers/page-analyze-worker.js

@@ -0,0 +1,872 @@
+"use strict";
+// SPDX-FileCopyrightText: 2025-2026 Reftrix Contributors
+// SPDX-License-Identifier: AGPL-3.0-only
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EMBEDDING_CHUNK_SIZE = exports.HTML_HUGE_THRESHOLD = exports.HTML_LARGE_THRESHOLD = exports.MEMORY_CRITICAL_THRESHOLD_MB = exports.MEMORY_DEGRADATION_THRESHOLD_MB = exports.DEFAULT_LOCK_EXTEND_INTERVAL = exports.DEFAULT_LOCK_DURATION = exports.extendJobLock = exports.createLockExtender = exports.generateJsAnimationTextRepresentation = exports.createPhaseProgressInterpolator = exports.tryGarbageCollect = exports.checkMemoryPressure = void 0;
+exports.createPageAnalyzeWorker = createPageAnalyzeWorker;
+exports.processPageAnalyzeJob = processPageAnalyzeJob;
+/**
+ * PageAnalyzeWorker - BullMQ Worker for Async Page Analysis
+ *
+ * Phase3-2: Handles heavy page analysis jobs asynchronously.
+ * Designed for WebGL-heavy sites (Linear, Vercel, Notion) that may timeout
+ * in synchronous processing.
+ *
+ * Configuration:
+ * - concurrency: 1 (singleton browser, avoid race condition)
+ * - lockDuration: 2400000ms (40 min, extended for CPU-bound embedding phase), configurable via BULLMQ_LOCK_DURATION
+ * - attempts: 1 (no retries for WebGL sites)
+ *
+ * Lock Extension Strategy (Hybrid Approach):
+ * BullMQ v5.x provides automatic lock renewal via lockRenewTime (default: lockDuration/2).
+ * However, CPU-bound processing (e.g., Ollama Vision 10.7B) may block the event loop,
+ * preventing timer-based renewal. This module adds:
+ * 1. createLockExtender: setInterval-based periodic lock extension (secondary protection)
+ * 2. extendJobLock: explicit lock extension at async phase boundaries
+ * Together they provide dual-layer stall prevention for long-running jobs (30+ minutes).
+ *
+ * Architecture (v0.1.10+, TDA-C1 refactoring):
+ * This file is a thin orchestrator. Phase logic lives in ./phases/:
+ *   phase-0-ingest.ts  — HTML ingest + WebPage DB save
+ *   phase-1-layout.ts  — Layout Analysis + Part Extraction
+ *   phase-2-motion.ts  — Scroll Vision + Motion Detection + Scroll Vision Analysis
+ *   phase-3-quality.ts — Quality Evaluation
+ *   phase-4-narrative.ts — Narrative Analysis + Responsive Analysis
+ *   phase-5-embedding.ts — Embedding Generation (text + visual)
+ * Shared types/constants/helpers are in ./phases/types.ts.
+ *
+ * Environment Variables:
+ * - BULLMQ_LOCK_DURATION: Lock duration in ms (default: 2400000)
+ * - BULLMQ_LOCK_EXTEND_INTERVAL_MS: Lock extend interval in ms (default: 300000)
+ *
+ * @module workers/page-analyze-worker
+ */
+const bullmq_1 = require("bullmq");
+const redis_1 = require("../config/redis");
+const page_analyze_queue_1 = require("../queues/page-analyze-queue");
+const execution_status_tracker_1 = require("../tools/page/handlers/execution-status-tracker");
+const logger_1 = require("../utils/logger");
+const database_1 = require("@reftrixmcp/database");
+// Service handlers (same as used in page.analyze synchronous mode)
+const layout_handler_1 = require("../tools/page/handlers/layout-handler");
+const motion_handler_1 = require("../tools/page/handlers/motion-handler");
+const quality_handler_1 = require("../tools/page/handlers/quality-handler");
+const page_ingest_adapter_1 = require("../services/page-ingest-adapter");
+const background_design_db_service_1 = require("../services/background/background-design-db.service");
+const narrative_handler_1 = require("../tools/page/handlers/narrative-handler");
+// Scroll Vision Smart Capture
+const scroll_vision_capture_service_1 = require("../services/vision/scroll-vision-capture.service");
+const scroll_vision_analyzer_1 = require("../services/vision/scroll-vision.analyzer");
+const scroll_vision_persistence_service_1 = require("../services/vision/scroll-vision-persistence.service");
+// GPU Resource Manager: Vision/Embedding間のGPU動的切り替え
+const gpu_resource_manager_1 = require("../services/gpu-resource-manager");
+// Responsive Analysis
+const responsive_1 = require("../services/responsive");
+const url_validator_1 = require("../utils/url-validator");
+const core_1 = require("@reftrixmcp/core");
+// EmbeddingService singleton for GPU provider switching (switchProvider/releaseGpu)
+const ml_1 = require("@reftrixmcp/ml");
+// DB保存ロジック（SectionPattern, MotionPattern, QualityEvaluation, JSAnimationPattern）
+const worker_db_save_service_1 = require("../services/worker-db-save.service");
+// Section Merge/Split Post-Processor（過剰分割修正 + 巨大セクション再分割）
+const section_postprocessor_service_1 = require("../services/page/section-postprocessor.service");
+// Embedding generation (reuse from synchronous flow)
+const embedding_handler_1 = require("../tools/page/handlers/embedding-handler");
+const layout_embedding_service_1 = require("../services/layout-embedding.service");
+const frame_embedding_service_1 = require("../services/motion/frame-embedding.service");
+// Worker Memory Self-Monitoring（OOM防止用）
+const worker_memory_monitor_service_1 = require("../services/worker-memory-monitor.service");
+// Part Extraction (Phase 1.1)
+const part_extraction_service_1 = require("../services/part/part-extraction.service");
+const part_db_service_1 = require("../services/part/part-db.service");
+// Dynamic memory thresholds: lazy initialization via initMemoryConstants()
+// Stall recovery: BullMQ stalled event handler + periodic check
+const worker_stall_recovery_service_1 = require("../services/worker-stall-recovery.service");
+const page_analyze_queue_2 = require("../queues/page-analyze-queue");
+// SEC-M2: 安全な環境変数パース
+const safe_parse_int_1 = require("../utils/safe-parse-int");
+// Post-embedding backfill: DB-driven embedding gap detection and repair
+const embedding_backfill_service_1 = require("../services/embedding-backfill.service");
+// Frame Analysis DB保存ヘルパー（同期/非同期モード共有）
+const frame_analysis_save_helper_1 = require("../services/motion/frame-analysis-save.helper");
+// ============================================================================
+// Phase Module Imports (TDA-C1 refactoring)
+// ============================================================================
+const types_1 = require("./phases/types");
+Object.defineProperty(exports, "DEFAULT_LOCK_DURATION", { enumerable: true, get: function () { return types_1.DEFAULT_LOCK_DURATION; } });
+Object.defineProperty(exports, "DEFAULT_LOCK_EXTEND_INTERVAL", { enumerable: true, get: function () { return types_1.DEFAULT_LOCK_EXTEND_INTERVAL; } });
+Object.defineProperty(exports, "MEMORY_DEGRADATION_THRESHOLD_MB", { enumerable: true, get: function () { return types_1.MEMORY_DEGRADATION_THRESHOLD_MB; } });
+Object.defineProperty(exports, "MEMORY_CRITICAL_THRESHOLD_MB", { enumerable: true, get: function () { return types_1.MEMORY_CRITICAL_THRESHOLD_MB; } });
+Object.defineProperty(exports, "HTML_LARGE_THRESHOLD", { enumerable: true, get: function () { return types_1.HTML_LARGE_THRESHOLD; } });
+Object.defineProperty(exports, "HTML_HUGE_THRESHOLD", { enumerable: true, get: function () { return types_1.HTML_HUGE_THRESHOLD; } });
+Object.defineProperty(exports, "EMBEDDING_CHUNK_SIZE", { enumerable: true, get: function () { return types_1.EMBEDDING_CHUNK_SIZE; } });
+Object.defineProperty(exports, "checkMemoryPressure", { enumerable: true, get: function () { return types_1.checkMemoryPressure; } });
+Object.defineProperty(exports, "tryGarbageCollect", { enumerable: true, get: function () { return types_1.tryGarbageCollect; } });
+Object.defineProperty(exports, "createLockExtender", { enumerable: true, get: function () { return types_1.createLockExtender; } });
+Object.defineProperty(exports, "extendJobLock", { enumerable: true, get: function () { return types_1.extendJobLock; } });
+Object.defineProperty(exports, "createPhaseProgressInterpolator", { enumerable: true, get: function () { return types_1.createPhaseProgressInterpolator; } });
+Object.defineProperty(exports, "generateJsAnimationTextRepresentation", { enumerable: true, get: function () { return types_1.generateJsAnimationTextRepresentation; } });
+const phase_0_ingest_1 = require("./phases/phase-0-ingest");
+const phase_1_layout_1 = require("./phases/phase-1-layout");
+const phase_2_motion_1 = require("./phases/phase-2-motion");
+const phase_3_quality_1 = require("./phases/phase-3-quality");
+const phase_4_narrative_1 = require("./phases/phase-4-narrative");
+const phase_5_embedding_1 = require("./phases/phase-5-embedding");
+// ============================================================================
+// Embedding DI factories initialization
+// ============================================================================
+// Worker runs in a separate process; factories must be set before use
+// Single shared ONNX session to prevent memory leak from repeated LayoutEmbeddingService creation
+// P0-1: All embedding sub-phases (Section, Motion, Background, JSAnimation) share this singleton
+const sharedLayoutEmbeddingService = new layout_embedding_service_1.LayoutEmbeddingService();
+(0, layout_embedding_service_1.setEmbeddingServiceFactory)(() => ml_1.embeddingService);
+(0, layout_embedding_service_1.setPrismaClientFactory)(() => database_1.prisma);
+(0, embedding_handler_1.setBackgroundEmbeddingServiceFactory)(() => sharedLayoutEmbeddingService);
+(0, embedding_handler_1.setMotionLayoutEmbeddingServiceFactory)(() => sharedLayoutEmbeddingService);
+(0, embedding_handler_1.setBackgroundPrismaClientFactory)(() => database_1.prisma);
+(0, frame_embedding_service_1.setFramePrismaClientFactory)(() => database_1.prisma);
+// GPU Resource Manager: Vision/Embedding間のGPU動的切り替え (singleton)
+const gpuResourceManager = gpu_resource_manager_1.GpuResourceManager.getInstance();
+// ============================================================================
+// Pre-Return Pause: BullMQ moveToCompleted レースコンディション防止
+// ============================================================================
+//
+// BullMQ v5 の moveToCompleted Lua スクリプトは fetchNext=true の場合、
+// ジョブ完了と次のジョブ取得を1つのアトミック操作で行う。
+// これにより Worker.on('completed') イベントが発火する前に次のジョブが
+// active 状態に遷移してしまい、WorkerSupervisor の計画的再起動時に
+// 新規ジョブが「ブラウザ閉鎖済み」エラーで失敗するレースコンディションが発生する。
+//
+// 解決: Processor内で return 前に worker.pause(true) を呼ぶことで
+// BullMQ Worker.paused フラグを立て、fetchNext=false を保証する。
+// worker.pause(doNotWaitActive=true) はProcessor内から安全に呼べる。
+//
+// WorkerSupervisor側では job-completed IPC で再起動をトリガーする従来の
+// フローが維持され、shutdown処理中に新規ジョブが取得されることはない。
+// ============================================================================
+/**
+ * Module-level reference to the BullMQ Worker instance.
+ * Set by createPageAnalyzeWorker(), read by processPageAnalyzeJob().
+ * This bridge enables the Processor to call worker.pause() before returning.
+ */
+let _workerInstanceRef = null;
+/**
+ * Whether pre-return pause is enabled (maxJobsBeforeRestart > 0).
+ * Read from WORKER_MAX_JOBS_BEFORE_RESTART env var (default: 1).
+ * When 0, pre-return pause is disabled (unlimited jobs per process).
+ */
+const _preReturnPauseEnabled = (0, safe_parse_int_1.safeParseInt)(process.env.WORKER_MAX_JOBS_BEFORE_RESTART, 1) > 0;
+// Connect gpuModeSignal to the @reftrixmcp/ml EmbeddingService singleton.
+// When GpuResourceManager requests a provider switch, the ONNX pipeline is
+// disposed and re-initialized with the new execution provider (CPU/CUDA).
+// We use mlEmbeddingService directly because LayoutEmbeddingService wraps
+// IEmbeddingService which doesn't expose switchProvider/releaseGpu.
+gpu_resource_manager_1.gpuModeSignal.onProviderSwitch = async (provider) => {
+    if (provider === "cuda") {
+        await ml_1.embeddingService.switchProvider("cuda");
+    }
+    else {
+        await ml_1.embeddingService.releaseGpu();
+    }
+};
+// ============================================================================
+// Dynamic Memory Configuration (lazy initialization via initMemoryConstants)
+// ============================================================================
+// L-3 fix: moved from module-level resolveMemoryConfig() to lazy init.
+// Constants in types.ts are updated on first call to initMemoryConstants().
+(0, types_1.initMemoryConstants)();
+// ============================================================================
+// processPageAnalyzeJob — Thin Orchestrator
+// ============================================================================
+/**
+ * Process a page analysis job by orchestrating phase modules.
+ *
+ * This function creates the pipeline state and context, then delegates
+ * to extracted phase modules (phase-0 through phase-5) in sequence.
+ * Phase 5 (Embedding) and post-embedding backfill remain inline because
+ * they reference module-level singletons (_workerInstanceRef, gpuResourceManager)
+ * and have a different API signature (EmbeddingPhaseParams + EmbeddingPhaseDeps).
+ *
+ * @param job - BullMQ job instance
+ * @param token - BullMQ worker token for lock management
+ * @returns Job result with completed/failed phases and analysis results
+ */
+async function processPageAnalyzeJob(job, token) {
+    const startTime = Date.now();
+    const { webPageId, url, options } = job.data;
+    // Lock extension: Create periodic lock extender as secondary protection
+    // BullMQ's built-in lockRenewTime (lockDuration/2) handles the primary case,
+    // but CPU-bound phases (Ollama Vision) may block the event loop.
+    const effectiveToken = token ?? job.token ?? "";
+    const effectiveLockDuration = types_1.DEFAULT_LOCK_DURATION;
+    const lockExtender = (0, types_1.createLockExtender)(job, effectiveToken, effectiveLockDuration, types_1.DEFAULT_LOCK_EXTEND_INTERVAL);
+    // Start lock extender before processing phases
+    lockExtender.start();
+    if ((0, logger_1.isDevelopment)()) {
+        logger_1.logger.info("[PageAnalyzeWorker] Processing job", {
+            jobId: job.id,
+            webPageId,
+            url,
+            options,
+            lockExtension: {
+                lockDuration: effectiveLockDuration,
+                extendInterval: types_1.DEFAULT_LOCK_EXTEND_INTERVAL,
+                hasToken: !!effectiveToken,
+            },
+        });
+    }
+    // Initialize status tracker for progress reporting
+    // Send detailed progress data including currentPhase and phases for SSE clients
+    const statusTracker = new execution_status_tracker_1.ExecutionStatusTrackerV2({
+        webPageId,
+        url,
+        onStatusChange: (status) => {
+            // Build detailed progress data for SSE consumers
+            const progressData = {
+                overallProgress: status.overallProgress,
+                currentPhase: status.currentPhase,
+                phases: status.phases,
+                webPageId: status.webPageId,
+                url: status.url,
+                startedAt: status.startedAt.toISOString(),
+                lastUpdatedAt: status.lastUpdatedAt.toISOString(),
+                estimatedCompletion: status.estimatedCompletion?.toISOString(),
+            };
+            job.updateProgress(progressData).catch((err) => {
+                logger_1.logger.warn("[PageAnalyzeWorker] Failed to update job progress", {
+                    jobId: job.id,
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            });
+        },
+    });
+    statusTracker.initialize();
+    // Create PhaseContext (immutable across phases)
+    const ctx = {
+        job,
+        options,
+        url,
+        webPageId,
+        effectiveToken,
+        effectiveLockDuration,
+        statusTracker,
+    };
+    // Create PipelineState (mutable, shared across phases)
+    const state = {
+        actualWebPageId: webPageId,
+        completedPhases: [],
+        failedPhases: [],
+        results: {},
+        layoutResultForNarrative: null,
+        sectionSaveResult: null,
+        motionSaveResult: null,
+        jsSaveResult: null,
+        bgSaveResult: null,
+        motionResultForEmbedding: null,
+        jsAnimationsForEmbedding: null,
+        scrollVisionSaveResult: null,
+        scrollVisionResultForEmbedding: null,
+        scrollVisionCapturesForDeferred: null,
+        html: null,
+        screenshotBase64: undefined,
+        narrativePreDisabled: false,
+        visionPreDisabled: false,
+        memoryAborted: false,
+    };
+    try {
+        // =====================================================
+        // Phase 0: Ingest (HTML取得) + Phase 0.5: WebPage DB Save
+        // =====================================================
+        const sharedBrowser = await (0, phase_0_ingest_1.processIngestPhase)(state, ctx, {
+            pageIngestAdapter: page_ingest_adapter_1.pageIngestAdapter,
+            prisma: database_1.prisma,
+        });
+        // =====================================================
+        // Phase 1: Layout Analysis + Phase 1.1: Part Extraction
+        // =====================================================
+        await (0, phase_1_layout_1.processLayoutPhase)(state, ctx, {
+            defaultAnalyzeLayout: layout_handler_1.defaultAnalyzeLayout,
+            saveBackgroundDesigns: background_design_db_service_1.saveBackgroundDesigns,
+            saveSectionPatterns: worker_db_save_service_1.saveSectionPatterns,
+            postProcessSections: section_postprocessor_service_1.postProcessSections,
+            extractPartsFromSection: part_extraction_service_1.extractPartsFromSection,
+            saveExtractedParts: part_db_service_1.saveExtractedParts,
+            prisma: database_1.prisma,
+        });
+        // =====================================================
+        // Ollama Vision Unload (1st point): After Phase 1 / before Phase 1.5
+        // Ollama VisionがVRAMにロードされたまま残る。
+        // Phase 1.5 (Scroll Capture) のChromium VRAM確保 + Phase 2.5 (Scroll Vision Analysis) の
+        // OllamaReadinessProbe VRAM閾値(8192MB)クリアのため、ここで解放する。
+        // Phase 2.5でVisionが必要な場合はOllamaが自動再ロードする。
+        // 冪等: useVision=falseでVision未ロード時もno-opで安全。
+        // =====================================================
+        await (0, types_1.unloadOllamaVisionModel)();
+        // =====================================================
+        // Phase 1.5 + 2 + 2.5: Motion (includes browser close + Ollama unload 2nd point)
+        // =====================================================
+        await (0, phase_2_motion_1.processMotionPhase)(state, ctx, sharedBrowser, {
+            captureScrollPositions: scroll_vision_capture_service_1.captureScrollPositions,
+            defaultDetectMotion: motion_handler_1.defaultDetectMotion,
+            analyzeScrollCaptures: scroll_vision_analyzer_1.analyzeScrollCaptures,
+            saveMotionPatterns: worker_db_save_service_1.saveMotionPatterns,
+            saveJsAnimationPatterns: worker_db_save_service_1.saveJsAnimationPatterns,
+            saveScrollVisionResults: scroll_vision_persistence_service_1.saveScrollVisionResults,
+            saveFrameAnalysisToDb: frame_analysis_save_helper_1.saveFrameAnalysisToDb,
+            gpuResourceManager,
+            pageIngestAdapter: page_ingest_adapter_1.pageIngestAdapter,
+            prisma: database_1.prisma,
+        });
+        // =====================================================
+        // Phase 3: Quality Evaluation
+        // =====================================================
+        await (0, phase_3_quality_1.processQualityPhase)(state, ctx, {
+            defaultEvaluateQuality: quality_handler_1.defaultEvaluateQuality,
+            saveQualityEvaluation: worker_db_save_service_1.saveQualityEvaluation,
+            saveQualityBenchmarks: worker_db_save_service_1.saveQualityBenchmarks,
+            buildQualityBenchmarkInputs: worker_db_save_service_1.buildQualityBenchmarkInputs,
+            prisma: database_1.prisma,
+        });
+        // =====================================================
+        // Phase 4 + 4.5: Narrative + Responsive (includes Ollama unload 3rd point)
+        // =====================================================
+        await (0, phase_4_narrative_1.processNarrativePhase)(state, ctx, {
+            handleNarrativeAnalysis: narrative_handler_1.handleNarrativeAnalysis,
+            responsiveAnalysisService: responsive_1.responsiveAnalysisService,
+            responsivePersistenceService: responsive_1.responsivePersistenceService,
+            validateExternalUrl: url_validator_1.validateExternalUrl,
+            isUrlAllowedByRobotsTxt: core_1.isUrlAllowedByRobotsTxt,
+        });
+        // =====================================================
+        // Memory Check 3: Before Phase 5 (Embedding)
+        // Even under memory pressure, we want to attempt embedding generation
+        // because it persists already-collected data to the DB.
+        // Just log a warning for observability.
+        // =====================================================
+        {
+            const memCheck3 = (0, types_1.checkMemoryPressure)();
+            if (memCheck3.shouldAbort) {
+                logger_1.logger.warn("[PageAnalyzeWorker] [Memory Critical] RSS high before embedding, attempting minimal save", {
+                    rssMb: memCheck3.rssMb,
+                    threshold: types_1.MEMORY_CRITICAL_THRESHOLD_MB,
+                    url,
+                });
+            }
+        }
+        // =====================================================
+        // Phase 5: Embedding Generation (delegated to processEmbeddingPhase)
+        // =====================================================
+        // Extend lock before Embedding phase
+        await (0, types_1.extendJobLock)(job, effectiveToken, effectiveLockDuration, "embedding");
+        const responsiveAnalysisIdForEmbedding = state.results?.responsive?.responsiveAnalysisId;
+        const embeddingEnabled = state.actualWebPageId &&
+            ((state.sectionSaveResult?.idMapping?.size ?? 0) +
+                (state.motionSaveResult?.idMapping?.size ?? 0) +
+                (state.jsSaveResult?.idMapping?.size ?? 0) +
+                (state.bgSaveResult?.idMapping?.size ?? 0) +
+                (state.scrollVisionSaveResult?.idMapping?.size ?? 0) >
+                0 ||
+                !!responsiveAnalysisIdForEmbedding ||
+                (state.results?.partExtraction?.totalPartsSaved ?? 0) > 0);
+        if (embeddingEnabled) {
+            // GPU Resource Manager: Acquire GPU for Embedding (unloads Ollama, switches ONNX to CUDA)
+            try {
+                const embeddingAcquireResult = await gpuResourceManager.acquireForEmbedding();
+                logger_1.logger.debug("[PageAnalyzeWorker] GPU acquired for embedding", {
+                    acquired: embeddingAcquireResult.acquired,
+                    fallbackToCpu: embeddingAcquireResult.fallbackToCpu,
+                });
+            }
+            catch (gpuError) {
+                logger_1.logger.warn("[PageAnalyzeWorker] GPU acquire for embedding failed, using CPU", {
+                    error: gpuError instanceof Error ? gpuError.message : String(gpuError),
+                });
+                // Continue with CPU mode - embedding will work, just slower
+            }
+            await job.updateProgress(types_1.PHASE_PROGRESS.EMBEDDING_START);
+            const embeddingPhaseResult = await (0, phase_5_embedding_1.processEmbeddingPhase)({
+                webPageId: state.actualWebPageId,
+                url,
+                job,
+                effectiveToken,
+                effectiveLockDuration,
+                sectionSaveResult: state.sectionSaveResult,
+                motionSaveResult: state.motionSaveResult,
+                jsSaveResult: state.jsSaveResult,
+                bgSaveResult: state.bgSaveResult,
+                scrollVisionSaveResult: state.scrollVisionSaveResult,
+                layoutResultForNarrative: state.layoutResultForNarrative,
+                motionResultForEmbedding: state.motionResultForEmbedding,
+                jsAnimationsForEmbedding: state.jsAnimationsForEmbedding,
+                scrollVisionResultForEmbedding: state.scrollVisionResultForEmbedding,
+                responsiveAnalysisId: responsiveAnalysisIdForEmbedding,
+                partsSavedCount: state.results?.partExtraction?.totalPartsSaved ?? 0,
+                screenshotBase64: state.screenshotBase64,
+                sharedBrowser,
+                onProgress: (0, types_1.createPhaseProgressInterpolator)(job, types_1.PHASE_PROGRESS.EMBEDDING_START, types_1.PHASE_PROGRESS.EMBEDDING_COMPLETE),
+            }, {
+                sharedLayoutEmbeddingService,
+                gpuResourceManager,
+                prisma: database_1.prisma,
+            });
+            // Release screenshotBase64 after Phase 5 (visual embedding) completes
+            // screenshotBase64はPhase 5のDINOv2 visual embedding完了後に解放
+            state.screenshotBase64 = undefined;
+            (0, types_1.tryGarbageCollect)();
+            // Map embedding phase result back to job results
+            // Safety: state.results is always initialized to {} in PipelineState construction (line ~315),
+            // but the type is optional because PageAnalyzeJobResult.results is optional.
+            // We assert non-null here since we know it's always set.
+            const results = state.results;
+            if (embeddingPhaseResult.sectionEmbeddingsGenerated > 0 ||
+                embeddingPhaseResult.sectionVisualEmbeddingsGenerated > 0 ||
+                embeddingPhaseResult.motionEmbeddingsGenerated > 0 ||
+                embeddingPhaseResult.bgEmbeddingsGenerated > 0 ||
+                embeddingPhaseResult.jsAnimationEmbeddingsGenerated > 0 ||
+                embeddingPhaseResult.responsiveEmbeddingsGenerated > 0 ||
+                embeddingPhaseResult.partEmbeddingsGenerated > 0 ||
+                embeddingPhaseResult.partVisualEmbeddingsGenerated > 0) {
+                const embeddingResult = {};
+                if (embeddingPhaseResult.sectionEmbeddingsGenerated > 0) {
+                    embeddingResult.sectionEmbeddingsGenerated =
+                        embeddingPhaseResult.sectionEmbeddingsGenerated;
+                }
+                if (embeddingPhaseResult.sectionVisualEmbeddingsGenerated > 0) {
+                    embeddingResult.sectionVisualEmbeddingsGenerated =
+                        embeddingPhaseResult.sectionVisualEmbeddingsGenerated;
+                }
+                if (embeddingPhaseResult.motionEmbeddingsGenerated > 0) {
+                    embeddingResult.motionEmbeddingsGenerated =
+                        embeddingPhaseResult.motionEmbeddingsGenerated;
+                }
+                if (embeddingPhaseResult.bgEmbeddingsGenerated > 0) {
+                    embeddingResult.backgroundDesignEmbeddingsGenerated =
+                        embeddingPhaseResult.bgEmbeddingsGenerated;
+                }
+                if (embeddingPhaseResult.jsAnimationEmbeddingsGenerated > 0) {
+                    embeddingResult.jsAnimationEmbeddingsGenerated =
+                        embeddingPhaseResult.jsAnimationEmbeddingsGenerated;
+                }
+                if (embeddingPhaseResult.responsiveEmbeddingsGenerated > 0) {
+                    embeddingResult.responsiveEmbeddingsGenerated =
+                        embeddingPhaseResult.responsiveEmbeddingsGenerated;
+                }
+                if (embeddingPhaseResult.partEmbeddingsGenerated > 0) {
+                    embeddingResult.partEmbeddingsGenerated = embeddingPhaseResult.partEmbeddingsGenerated;
+                }
+                if (embeddingPhaseResult.partVisualEmbeddingsGenerated > 0) {
+                    embeddingResult.partVisualEmbeddingsGenerated =
+                        embeddingPhaseResult.partVisualEmbeddingsGenerated;
+                }
+                results.embedding = embeddingResult;
+            }
+            if (embeddingPhaseResult.completed) {
+                state.completedPhases.push("embedding");
+            }
+            await job.updateProgress(types_1.PHASE_PROGRESS.EMBEDDING_COMPLETE);
+            // =====================================================
+            // Post-Embedding Backfill: Detect and repair missing embeddings
+            // If Phase 5 partially failed (OOM, memory pressure), some patterns
+            // may have been saved to DB but lack embeddings. This step reads
+            // from DB, generates embeddings in small chunks, and saves them back.
+            // =====================================================
+            {
+                // Release all in-memory refs to minimize RSS before backfill
+                // (backfill reads from DB, so pipeline data is no longer needed)
+                state.layoutResultForNarrative = null;
+                state.motionResultForEmbedding = null;
+                state.jsAnimationsForEmbedding = null;
+                state.scrollVisionResultForEmbedding = null;
+                (0, types_1.tryGarbageCollect)();
+                await (0, types_1.extendJobLock)(job, effectiveToken, effectiveLockDuration, "embedding-backfill");
+                const coverage = await (0, embedding_backfill_service_1.checkWebPageEmbeddingCoverage)(state.actualWebPageId);
+                const totalMissing = coverage.reduce((sum, c) => sum + c.missing, 0);
+                if (totalMissing > 0) {
+                    logger_1.logger.info("[PageAnalyzeWorker] Post-embedding backfill starting", {
+                        url,
+                        webPageId: state.actualWebPageId,
+                        totalMissing,
+                        coverage: coverage.map((c) => `${c.type}: ${c.embedded}/${c.total}`),
+                    });
+                    const backfillResult = await (0, embedding_backfill_service_1.backfillWebPageEmbeddings)(state.actualWebPageId, {
+                        chunkSize: 5,
+                        onProgress: (_type, _done, _total) => {
+                            // Extend lock on each progress update to prevent stall during backfill
+                            (0, types_1.extendJobLock)(job, effectiveToken, effectiveLockDuration, "backfill-progress").catch(() => { });
+                        },
+                    });
+                    logger_1.logger.info("[PageAnalyzeWorker] Post-embedding backfill completed", {
+                        url,
+                        totalBackfilled: backfillResult.totalBackfilled,
+                        sectionBackfilled: backfillResult.sectionBackfilled,
+                        motionBackfilled: backfillResult.motionBackfilled,
+                        backgroundBackfilled: backfillResult.backgroundBackfilled,
+                        jsAnimationBackfilled: backfillResult.jsAnimationBackfilled,
+                        responsiveBackfilled: backfillResult.responsiveBackfilled,
+                        errors: backfillResult.errors.length,
+                    });
+                    // Add backfill results to embedding phase counters
+                    if (embeddingPhaseResult) {
+                        embeddingPhaseResult.sectionEmbeddingsGenerated += backfillResult.sectionBackfilled;
+                        embeddingPhaseResult.motionEmbeddingsGenerated += backfillResult.motionBackfilled;
+                        embeddingPhaseResult.bgEmbeddingsGenerated += backfillResult.backgroundBackfilled;
+                        embeddingPhaseResult.jsAnimationEmbeddingsGenerated +=
+                            backfillResult.jsAnimationBackfilled;
+                        embeddingPhaseResult.responsiveEmbeddingsGenerated +=
+                            backfillResult.responsiveBackfilled;
+                        // Update results object with new totals
+                        if (backfillResult.totalBackfilled > 0) {
+                            if (!results.embedding) {
+                                results.embedding = {};
+                            }
+                            results.embedding.sectionEmbeddingsGenerated =
+                                embeddingPhaseResult.sectionEmbeddingsGenerated;
+                            results.embedding.motionEmbeddingsGenerated =
+                                embeddingPhaseResult.motionEmbeddingsGenerated;
+                            results.embedding.backgroundDesignEmbeddingsGenerated =
+                                embeddingPhaseResult.bgEmbeddingsGenerated;
+                            results.embedding.jsAnimationEmbeddingsGenerated =
+                                embeddingPhaseResult.jsAnimationEmbeddingsGenerated;
+                            results.embedding.responsiveEmbeddingsGenerated =
+                                embeddingPhaseResult.responsiveEmbeddingsGenerated;
+                            results.embedding.partEmbeddingsGenerated =
+                                embeddingPhaseResult.partEmbeddingsGenerated;
+                        }
+                    }
+                }
+                else {
+                    logger_1.logger.debug("[PageAnalyzeWorker] Post-embedding backfill: no missing embeddings", {
+                        url,
+                        webPageId: state.actualWebPageId,
+                    });
+                }
+            }
+        }
+        // GPU Resource Manager: Release GPU resources for next job's Vision phase
+        try {
+            await gpuResourceManager.release();
+        }
+        catch (gpuError) {
+            logger_1.logger.warn("[PageAnalyzeWorker] GPU release failed (non-fatal)", {
+                error: gpuError instanceof Error ? gpuError.message : String(gpuError),
+            });
+        }
+        // =====================================================
+        // Memory Cleanup: Release all remaining intermediate data
+        // All analysis and embedding phases are complete; release large
+        // objects before building the final result.
+        // =====================================================
+        {
+            const beforeRss = Math.round(process.memoryUsage().rss / 1024 / 1024);
+            state.layoutResultForNarrative = null;
+            state.motionResultForEmbedding = null;
+            state.scrollVisionResultForEmbedding = null;
+            state.jsAnimationsForEmbedding = null;
+            (0, types_1.tryGarbageCollect)();
+            if ((0, logger_1.isDevelopment)()) {
+                const afterRss = Math.round(process.memoryUsage().rss / 1024 / 1024);
+                logger_1.logger.debug("[PageAnalyzeWorker] [MemCleanup] Post-Embedding final cleanup", {
+                    beforeRssMb: beforeRss,
+                    afterRssMb: afterRss,
+                    reclaimedMb: beforeRss - afterRss,
+                    releasedRefs: [
+                        "layoutResultForNarrative",
+                        "motionResultForEmbedding",
+                        "scrollVisionResultForEmbedding",
+                        "jsAnimationsForEmbedding",
+                    ],
+                });
+            }
+        }
+        // =====================================================
+        // Finalize
+        // =====================================================
+        statusTracker.startPhase("finalizing");
+        statusTracker.completePhase("finalizing");
+        const processingTimeMs = Date.now() - startTime;
+        const success = state.failedPhases.length === 0;
+        const partialSuccess = !success && state.completedPhases.length > 0;
+        const result = {
+            webPageId: state.actualWebPageId, // v0.1.0: 実際のDB IDを返す
+            success,
+            partialSuccess,
+            completedPhases: state.completedPhases,
+            failedPhases: state.failedPhases,
+            processingTimeMs,
+            completedAt: new Date().toISOString(),
+        };
+        // Add results only if there are any (avoid undefined assignment with exactOptionalPropertyTypes)
+        if (state.results && Object.keys(state.results).length > 0) {
+            result.results = state.results;
+        }
+        if ((0, logger_1.isDevelopment)()) {
+            logger_1.logger.info("[PageAnalyzeWorker] Job completed", {
+                jobId: job.id,
+                requestedWebPageId: webPageId,
+                actualWebPageId: state.actualWebPageId,
+                success,
+                partialSuccess,
+                completedPhases: state.completedPhases,
+                failedPhases: state.failedPhases,
+                processingTimeMs,
+            });
+        }
+        // =====================================================
+        // Pre-return pause: fetchNext=false を保証してレースコンディション防止
+        // =====================================================
+        // BullMQ moveToCompleted Lua スクリプトは fetchNext=true だと
+        // ジョブ完了と同時に次ジョブを取得する。worker.pause(true) で
+        // Worker.paused フラグを立てることで fetchNext=false が保証され、
+        // WorkerSupervisor の計画的再起動が安全に実行できる。
+        if (_preReturnPauseEnabled && _workerInstanceRef) {
+            try {
+                await _workerInstanceRef.pause(true);
+                if ((0, logger_1.isDevelopment)()) {
+                    logger_1.logger.info("[PageAnalyzeWorker] Pre-return pause applied (fetchNext=false guaranteed)");
+                }
+            }
+            catch (pauseError) {
+                // pause失敗は致命的でない（WorkerSupervisor側のshutdownでフォールバック）
+                logger_1.logger.warn("[PageAnalyzeWorker] Pre-return pause failed (non-fatal)", {
+                    error: pauseError instanceof Error ? pauseError.message : String(pauseError),
+                });
+            }
+        }
+        // =====================================================
+        // Post-job memory self-check (SEC監査 Low #2 対応)
+        // =====================================================
+        // WorkerSupervisorがジョブカウントで再起動するが、
+        // メモリが閾値を超えた場合はワーカー自身でも graceful exit する。
+        // これによりOOMキラーによる強制終了を防止する。
+        // setImmediate で result を BullMQ に返却した後にチェックする。
+        setImmediate(() => {
+            (0, worker_memory_monitor_service_1.performMemoryCheckAndExit)();
+        });
+        return result;
+    }
+    catch (error) {
+        const processingTimeMs = Date.now() - startTime;
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        logger_1.logger.error("[PageAnalyzeWorker] Job failed with exception", {
+            jobId: job.id,
+            webPageId,
+            error: errorMessage,
+            processingTimeMs,
+        });
+        // Note: failure path では pause(true) を呼ばない。
+        // success path の pause は 'completed' → IPC 'job-completed' → 計画的再起動の
+        // フローで安全だが、failure path では 'failed' イベントが IPC を送信しないため、
+        // pause すると Worker が永久停止する。autorun: false で起動時レースは防止済み。
+        // Re-throw to let BullMQ record the failure
+        // Note: BullMQ will capture the error message from the thrown error
+        throw error;
+    }
+    finally {
+        // Always stop the lock extender to prevent leaked intervals
+        lockExtender.stop();
+        // SEC-L1: Defensive cleanup - release capture buffers on any exit path
+        state.scrollVisionCapturesForDeferred = null;
+    }
+}
+// ============================================================================
+// Worker Factory
+// ============================================================================
+/**
+ * Create a PageAnalyzeWorker instance
+ *
+ * @param options - Worker configuration options
+ * @returns Worker instance with lifecycle methods
+ */
+function createPageAnalyzeWorker(options = {}) {
+    const { redisConfig, concurrency = types_1.DEFAULT_CONCURRENCY, lockDuration = types_1.DEFAULT_LOCK_DURATION, verbose = (0, logger_1.isDevelopment)(), } = options;
+    const config = (0, redis_1.getRedisConfig)(redisConfig);
+    if (verbose) {
+        logger_1.logger.info("[PageAnalyzeWorker] Creating worker", {
+            queueName: page_analyze_queue_1.PAGE_ANALYZE_QUEUE_NAME,
+            concurrency,
+            lockDuration,
+            redisHost: config.host,
+            redisPort: config.port,
+        });
+    }
+    const worker = new bullmq_1.Worker(page_analyze_queue_1.PAGE_ANALYZE_QUEUE_NAME, processPageAnalyzeJob, {
+        connection: {
+            host: config.host,
+            port: config.port,
+            maxRetriesPerRequest: config.maxRetriesPerRequest,
+        },
+        // Explicit start from start-workers.ts after local initialization is complete.
+        autorun: false,
+        concurrency,
+        lockDuration,
+        // Stalled job settings (detect stuck jobs)
+        // stalledInterval = lockDuration/4 to avoid false stall detection during legitimate long processing
+        stalledInterval: Math.max(60000, Math.floor(lockDuration / 4)),
+        maxStalledCount: 3, // Allow 3 stalls before failing (CPU-bound embedding phase may block event loop)
+    });
+    // Set module-level reference for Processor→Worker bridge (pre-return pause)
+    _workerInstanceRef = worker;
+    // SEC-M1: Pre-return pause は concurrency=1 前提の設計。
+    // concurrency > 1 では複数 Processor が同一 Worker に対して pause を呼ぶ可能性がある。
+    // BullMQ の pause() は冪等であるため安全だが、設計意図として警告を出す。
+    if (concurrency > 1 && _preReturnPauseEnabled) {
+        logger_1.logger.warn("[PageAnalyzeWorker] Pre-return pause is designed for concurrency=1. " +
+            "concurrency > 1 may cause unexpected pause timing.", { concurrency });
+    }
+    // Event handlers for monitoring
+    worker.on("completed", (job, result) => {
+        if (verbose) {
+            logger_1.logger.info("[PageAnalyzeWorker] Job completed event", {
+                jobId: job.id,
+                webPageId: result.webPageId,
+                success: result.success,
+                partialSuccess: result.partialSuccess,
+            });
+        }
+        // P1-D: Notify parent process (WorkerSupervisor) of job completion via IPC
+        // This enables maxJobsBeforeRestart planned restarts for OOM prevention
+        try {
+            process.send?.({ type: "job-completed", jobId: job.id });
+        }
+        catch {
+            // IPC channel may be closed if parent is shutting down; non-fatal
+        }
+    });
+    worker.on("failed", (job, error) => {
+        logger_1.logger.error("[PageAnalyzeWorker] Job failed event", {
+            jobId: job?.id,
+            error: error.message,
+        });
+    });
+    worker.on("error", (error) => {
+        logger_1.logger.error("[PageAnalyzeWorker] Worker error", {
+            error: error.message,
+        });
+    });
+    // Stall recovery: Create a Queue instance for job access during stall handling
+    const recoveryQueue = (0, page_analyze_queue_2.createPageAnalyzeQueue)(redisConfig);
+    // Build StalledJobAccessor for handleStalledJob DI
+    const stalledJobAccessor = {
+        getJob: async (stalledJobId) => {
+            const job = await recoveryQueue.getJob(stalledJobId);
+            if (!job || !job.id)
+                return null;
+            return {
+                id: job.id,
+                progress: typeof job.progress === "number" ? job.progress : 0,
+                processedOn: job.processedOn,
+                data: {
+                    webPageId: job.data?.webPageId ?? "",
+                    url: job.data?.url ?? "",
+                },
+                moveToFailed: async (err, token, fetchNext) => {
+                    await job.moveToFailed(err, token, fetchNext);
+                },
+                moveToCompleted: async (returnValue, token, fetchNext) => {
+                    await job.moveToCompleted(returnValue, token, fetchNext);
+                },
+                getState: async () => job.getState(),
+            };
+        },
+    };
+    // Enhanced stalled event handler: trigger custom recovery
+    worker.on("stalled", (jobId) => {
+        logger_1.logger.warn("[PageAnalyzeWorker] Job stalled — triggering recovery", { jobId });
+        // Fire-and-forget: recovery runs asynchronously, errors are logged inside handleStalledJob
+        (0, worker_stall_recovery_service_1.handleStalledJob)(jobId, stalledJobAccessor)
+            .then((result) => {
+            if (result.success) {
+                logger_1.logger.info("[PageAnalyzeWorker] Stalled job recovery result", {
+                    jobId: result.jobId,
+                    action: result.action,
+                    category: result.category,
+                });
+            }
+        })
+            .catch((err) => {
+            const msg = err instanceof Error ? err.message : String(err);
+            logger_1.logger.error("[PageAnalyzeWorker] Stalled job recovery error", { jobId, error: msg });
+        });
+    });
+    // Build DI functions for periodic stall check (reuse from recoverOrphanedJobs pattern)
+    const getActiveJobsFn = async () => {
+        const activeJobs = await recoveryQueue.getJobs(["active"], 0, 100);
+        return activeJobs
+            .filter((job) => job.id !== undefined)
+            .map((job) => ({
+            jobId: job.id ?? "",
+            state: "active",
+            progress: typeof job.progress === "number" ? job.progress : 0,
+            processedOn: job.processedOn,
+            lockDurationMs: lockDuration,
+            data: {
+                webPageId: job.data?.webPageId ?? "",
+                url: job.data?.url ?? "",
+            },
+        }));
+    };
+    const moveToFailedFn = async (failJobId, reason) => {
+        const job = await recoveryQueue.getJob(failJobId);
+        if (job) {
+            await job.moveToFailed(new Error(reason), "0", false);
+        }
+    };
+    const moveToCompletedFn = async (completeJobId) => {
+        const job = await recoveryQueue.getJob(completeJobId);
+        if (job) {
+            await job.moveToCompleted({
+                webPageId: job.data?.webPageId ?? "",
+                success: true,
+                partialSuccess: true,
+                completedPhases: [],
+                failedPhases: [],
+                processingTimeMs: 0,
+                completedAt: new Date().toISOString(),
+            }, "0", false);
+        }
+    };
+    // Startup recovery: recover orphaned jobs from previous crash/restart
+    (0, worker_stall_recovery_service_1.recoverOrphanedJobs)(getActiveJobsFn, moveToFailedFn, moveToCompletedFn, lockDuration)
+        .then((result) => {
+        if (result.recoveredCount > 0) {
+            logger_1.logger.info("[PageAnalyzeWorker] Startup recovery completed", {
+                recoveredCount: result.recoveredCount,
+                failedCount: result.failedCount,
+            });
+        }
+    })
+        .catch((err) => {
+        const msg = err instanceof Error ? err.message : String(err);
+        logger_1.logger.warn("[PageAnalyzeWorker] Startup recovery failed (non-fatal)", { error: msg });
+    });
+    // Periodic stall check: independent of BullMQ's internal stalledInterval
+    const periodicCheck = (0, worker_stall_recovery_service_1.createPeriodicStallCheck)(getActiveJobsFn, moveToFailedFn, moveToCompletedFn, { lockDurationMs: lockDuration });
+    let isRunning = true;
+    return {
+        worker,
+        close: async () => {
+            if (verbose) {
+                logger_1.logger.info("[PageAnalyzeWorker] Closing worker");
+            }
+            isRunning = false;
+            periodicCheck.stop();
+            // Release GPU resources before closing worker
+            try {
+                await gpuResourceManager.release();
+            }
+            catch {
+                // Release failure during shutdown is non-fatal
+            }
+            await recoveryQueue.close();
+            await worker.close();
+        },
+        pause: async () => {
+            if (verbose) {
+                logger_1.logger.info("[PageAnalyzeWorker] Pausing worker (no new jobs will be accepted)");
+            }
+            await worker.pause();
+        },
+        isRunning: () => isRunning,
+    };
+}
+//# sourceMappingURL=page-analyze-worker.js.map