blun-king-cli 9.1.587 → 9.1.588
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -185
- package/LIESMICH.txt +51 -13
- package/README.md +44 -47
- package/agent-spine-plugin/.codex-plugin/plugin.json +16 -4
- package/agent-spine-plugin/CHANGELOG.md +37 -5
- package/agent-spine-plugin/README.md +3 -3
- package/agent-spine-plugin/blun.plugin.json +45 -10
- package/agent-spine-plugin/docs/artifact-evaluation.md +93 -0
- package/agent-spine-plugin/docs/host-integration.md +42 -27
- package/agent-spine-plugin/docs/preflight-recall.md +4 -2
- package/agent-spine-plugin/docs/session-timeline.md +97 -236
- package/agent-spine-plugin/docs/world-model.md +25 -0
- package/agent-spine-plugin/hooks/codex.json +1 -1
- package/agent-spine-plugin/hooks/hooks.json +1 -1
- package/agent-spine-plugin/package.json +1 -3
- package/agent-spine-plugin/scripts/check-hosts.js +3 -3
- package/agent-spine-plugin/scripts/release-check.js +10 -5
- package/agent-spine-plugin/scripts/run-checks.js +4 -1
- package/agent-spine-plugin/scripts/run-tests-hermetic.js +32 -6
- package/agent-spine-plugin/src/cli-learning.js +15 -0
- package/agent-spine-plugin/src/cli.js +2 -0
- package/agent-spine-plugin/src/hook.js +32 -32
- package/agent-spine-plugin/src/lib/action-lesson-recall.js +73 -8
- package/agent-spine-plugin/src/lib/briefing.js +146 -36
- package/agent-spine-plugin/src/lib/channel-continuity.js +19 -0
- package/agent-spine-plugin/src/lib/delivery-agent-usage.js +14 -7
- package/agent-spine-plugin/src/lib/gateway-group-response.js +128 -0
- package/agent-spine-plugin/src/lib/gateway-runs.js +24 -15
- package/agent-spine-plugin/src/lib/hook-briefing-use.js +13 -3
- package/agent-spine-plugin/src/lib/hook-context.js +16 -3
- package/agent-spine-plugin/src/lib/hook-output.js +129 -5
- package/agent-spine-plugin/src/lib/hook-timeline.js +5 -3
- package/agent-spine-plugin/src/lib/indexed-memory.js +2 -2
- package/agent-spine-plugin/src/lib/learning-artifact-evaluator.js +114 -0
- package/agent-spine-plugin/src/lib/learning-context.js +11 -4
- package/agent-spine-plugin/src/lib/learning-measurements.js +2 -2
- package/agent-spine-plugin/src/lib/mcp-runtime.js +89 -3
- package/agent-spine-plugin/src/lib/mcp-source-context.js +12 -2
- package/agent-spine-plugin/src/lib/mcp-timeline-tools.js +91 -8
- package/agent-spine-plugin/src/lib/mcp-world-tools.js +2 -2
- package/agent-spine-plugin/src/lib/owned-file-lock.js +20 -1
- package/agent-spine-plugin/src/lib/persona-runtime.js +2 -2
- package/agent-spine-plugin/src/lib/preflight-delivery-id.js +27 -0
- package/agent-spine-plugin/src/lib/preflight.js +4 -4
- package/agent-spine-plugin/src/lib/session-timeline-codex.js +15 -0
- package/agent-spine-plugin/src/lib/session-timeline-contract.js +12 -4
- package/agent-spine-plugin/src/lib/session-timeline-event-extract.js +36 -7
- package/agent-spine-plugin/src/lib/session-timeline-host-origin.js +13 -10
- package/agent-spine-plugin/src/lib/session-timeline-invocation.js +1 -1
- package/agent-spine-plugin/src/lib/session-timeline-king.js +14 -0
- package/agent-spine-plugin/src/lib/session-timeline-prior.js +18 -12
- package/agent-spine-plugin/src/lib/session-timeline-provider.js +5 -0
- package/agent-spine-plugin/src/lib/session-timeline-query.js +2 -0
- package/agent-spine-plugin/src/lib/session-timeline-results.js +35 -10
- package/agent-spine-plugin/src/lib/session-timeline-source-open.js +30 -0
- package/agent-spine-plugin/src/lib/session-timeline.js +122 -75
- package/agent-spine-plugin/src/lib/source-roots.js +3 -2
- package/agent-spine-plugin/src/lib/task-knowledge-context.js +22 -1
- package/agent-spine-plugin/src/lib/timeline-continuation-update.js +100 -0
- package/agent-spine-plugin/src/lib/timeline-tool-guard.js +30 -7
- package/agent-spine-plugin/src/lib/timeline-user-feedback.js +217 -0
- package/agent-spine-plugin/src/lib/timeline-world-capture.js +233 -0
- package/agent-spine-plugin/src/lib/world-knowledge.js +59 -2
- package/agent-spine-plugin/src/lib/world-model.js +64 -9
- package/agent-spine-plugin/src/worker.js +13 -1
- package/bin/blun.js +43 -28
- package/bin/core-bootstrap.js +5 -4
- package/bin/king.js +43 -28
- package/bin/launcher-mode.js +1 -10
- package/bin/launcher-runtime.js +128 -295
- package/bin/managed-node.js +0 -0
- package/bin/managed-plugin-selection.cjs +0 -1
- package/bin/native-module-repair.js +0 -0
- package/bin/node-runtime.js +0 -0
- package/bin/node-version.js +0 -0
- package/bin/plugin-bootstrap.js +56 -120
- package/bin/private-paths.js +11 -34
- package/bin/standard-tools-bootstrap.js +34 -114
- package/bin/turn-thinking-policy.cjs +3 -11
- package/bin/update-copy.js +200 -0
- package/bin/update-lease.js +0 -0
- package/bin/update-notice.js +136 -289
- package/bin/verify-agent-behavior.cjs +122 -0
- package/bin/verify-agent-components.cjs +104 -0
- package/bin/verify-bundled-agent-sources.cjs +57 -0
- package/blun.mjs +143076 -135288
- package/bundled-agent-sources.json +701 -0
- package/package.json +12 -15
- package/standard-skills/translate-native/README.md +1293 -0
- package/standard-skills/translate-native/SKILL.md +172 -22
- package/standard-skills/translate-native/VERSION +1 -1
- package/standard-skills/translate-native/agents/openai.yaml +18 -0
- package/standard-skills/translate-native/assets/icon.svg +8 -0
- package/standard-skills/translate-native/docs/BLUN_CODE_INTEGRATION.md +76 -0
- package/standard-skills/translate-native/docs/PREMORTEM.md +489 -0
- package/standard-skills/translate-native/docs/WEBSITE_LOCALIZATION.md +2035 -0
- package/standard-skills/translate-native/docs/WEBSITE_LOCALIZATION_API.md +1302 -0
- package/standard-skills/translate-native/docs/WEBSITE_LOCALIZATION_EVIDENCE_HTTP.md +136 -0
- package/standard-skills/translate-native/docs/WEBSITE_LOCALIZATION_HEALTH_HTTP.md +130 -0
- package/standard-skills/translate-native/docs/WEBSITE_LOCALIZATION_HTTP_PROVIDER.md +175 -0
- package/standard-skills/translate-native/docs/WEBSITE_LOCALIZATION_RECEIPT_VERIFIER_HTTP.md +86 -0
- package/standard-skills/translate-native/integrations/AGENT_RULES.md +32 -0
- package/standard-skills/translate-native/integrations/adapters/blun-code-language-guard.js +514 -0
- package/standard-skills/translate-native/integrations/adapters/node-language-guard.js +230 -0
- package/standard-skills/translate-native/integrations/audit_log.py +327 -0
- package/standard-skills/translate-native/integrations/claude_language_hook.js +1536 -0
- package/standard-skills/translate-native/integrations/commercial_localization_profile.py +42 -0
- package/standard-skills/translate-native/integrations/delivery-policy.example.json +28 -0
- package/standard-skills/translate-native/integrations/enforced_delivery.py +543 -0
- package/standard-skills/translate-native/integrations/guard_service.py +435 -0
- package/standard-skills/translate-native/integrations/language_gateway.py +67 -0
- package/standard-skills/translate-native/integrations/mcp_auth_headers.py +198 -0
- package/standard-skills/translate-native/integrations/mcp_http_gateway.py +429 -0
- package/standard-skills/translate-native/integrations/non_language_html_entities.js +1485 -0
- package/standard-skills/translate-native/integrations/pre_output_guard.py +65 -0
- package/standard-skills/translate-native/integrations/task_router.py +101 -0
- package/standard-skills/translate-native/integrations/website_localization.py +401 -0
- package/standard-skills/translate-native/integrations/website_localization_api.py +581 -0
- package/standard-skills/translate-native/integrations/website_localization_benchmark.py +1885 -0
- package/standard-skills/translate-native/integrations/website_localization_benchmark_campaign.py +1772 -0
- package/standard-skills/translate-native/integrations/website_localization_benchmark_candidate.py +506 -0
- package/standard-skills/translate-native/integrations/website_localization_benchmark_http.py +400 -0
- package/standard-skills/translate-native/integrations/website_localization_benchmark_review_store.py +781 -0
- package/standard-skills/translate-native/integrations/website_localization_benchmark_reviewer_http.py +500 -0
- package/standard-skills/translate-native/integrations/website_localization_benchmark_runtime.py +1107 -0
- package/standard-skills/translate-native/integrations/website_localization_benchmark_suite.py +463 -0
- package/standard-skills/translate-native/integrations/website_localization_cms.py +2835 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_client.py +875 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_dispatch.py +805 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_http.py +588 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_lifecycle_monitor.py +991 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_receiver.py +1441 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_receiver_runtime.py +414 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_receiver_store.py +1073 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_removal_dispatch.py +865 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_client.py +583 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery.py +964 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_delivery_runtime.py +665 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_http.py +1153 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_runtime.py +675 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_source_service.py +1125 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_terminal_notification.py +674 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_terminal_notification_http.py +444 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_terminal_notification_receiver.py +1469 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_terminal_notification_receiver_runtime.py +1142 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_terminal_processing_monitor.py +634 -0
- package/standard-skills/translate-native/integrations/website_localization_cms_terminal_receiver_client.py +804 -0
- package/standard-skills/translate-native/integrations/website_localization_deepl_baseline.py +922 -0
- package/standard-skills/translate-native/integrations/website_localization_evidence_http.py +482 -0
- package/standard-skills/translate-native/integrations/website_localization_health.py +1541 -0
- package/standard-skills/translate-native/integrations/website_localization_health_http.py +372 -0
- package/standard-skills/translate-native/integrations/website_localization_http_provider.py +297 -0
- package/standard-skills/translate-native/integrations/website_localization_native_reference_http.py +479 -0
- package/standard-skills/translate-native/integrations/website_localization_native_reference_intake.py +363 -0
- package/standard-skills/translate-native/integrations/website_localization_native_reference_queue.py +1449 -0
- package/standard-skills/translate-native/integrations/website_localization_native_reference_store.py +420 -0
- package/standard-skills/translate-native/integrations/website_localization_quality_profiles.py +235 -0
- package/standard-skills/translate-native/integrations/website_localization_queue.py +671 -0
- package/standard-skills/translate-native/integrations/website_localization_receipt_verifier_http.py +516 -0
- package/standard-skills/translate-native/integrations/website_localization_release.py +928 -0
- package/standard-skills/translate-native/integrations/website_localization_release_coordinator.py +1008 -0
- package/standard-skills/translate-native/integrations/website_localization_runner.py +276 -0
- package/standard-skills/translate-native/integrations/website_localization_runtime.py +862 -0
- package/standard-skills/translate-native/integrations/website_localization_service.py +350 -0
- package/standard-skills/translate-native/integrations/website_localization_supervisor.py +511 -0
- package/standard-skills/translate-native/integrations/website_localization_worker.py +663 -0
- package/standard-skills/translate-native/provenance.json +3 -4
- package/standard-skills/translate-native/references/commercial-localization.md +177 -0
- package/standard-skills/translate-native/scripts/blun_language_guard.py +7 -1
- package/standard-skills/translate-native/scripts/check_commercial_review.py +80 -0
- package/standard-skills/translate-native/scripts/commercial_localization_profile.py +333 -0
- package/standard-tools/language-guard/LICENSE +21 -0
- package/standard-tools/language-guard/VERSION +1 -0
- package/standard-tools/language-guard/blun_language_guard.py +7 -1
- package/standard-tools/language-guard/check_commercial_review.py +80 -0
- package/standard-tools/language-guard/commercial_localization_profile.py +333 -0
- package/standard-tools/language-guard/language_gateway.py +62 -0
- package/standard-tools/language-guard/pre_output_guard.py +64 -0
- package/standard-tools/language-guard/provenance.json +4 -11
- package/standard-tools/manifest.json +34 -11
- package/telegram-plugin/commands/access.md +2 -10
- package/telegram-plugin/dist/bridge.mjs +64041 -687
- package/telegram-plugin/dist/mcp-server.mjs +72810 -9027
- package/telegram-plugin/dist/noise.mjs +28 -63511
- package/agent-spine-plugin/CONTRIBUTING.md +0 -52
- package/agent-spine-plugin/SECURITY.md +0 -47
- package/agent-spine-plugin/docs/assignment-continuation.md +0 -48
- package/agent-spine-plugin/docs/releasing.md +0 -85
- package/agent-spine-plugin/docs/structured-completion.md +0 -67
- package/bin/abort-listener-policy.cjs +0 -43
- package/bin/active-steer-priority-policy.cjs +0 -24
- package/bin/agent-api-http-adapter.mjs +0 -446
- package/bin/agent-api-private-http-server.mjs +0 -288
- package/bin/agent-api-runtime.mjs +0 -252
- package/bin/agent-api-service-environment.mjs +0 -236
- package/bin/agent-api-service-host.mjs +0 -209
- package/bin/agent-api-service-process.mjs +0 -171
- package/bin/agent-api-session-registry.mjs +0 -428
- package/bin/agent-api-tool-broker.cjs +0 -248
- package/bin/agent-api-turn-controller.mjs +0 -461
- package/bin/agent-api-usage-journal.cjs +0 -259
- package/bin/agent-resume-snapshot.cjs +0 -241
- package/bin/agentspine-king-goal-inbox.mjs +0 -111
- package/bin/agentspine-king-goal-intake.mjs +0 -106
- package/bin/approval-rejection-stop.cjs +0 -15
- package/bin/assistant-message-offload-policy.cjs +0 -284
- package/bin/baseline-skill-performance-policy.cjs +0 -39
- package/bin/bash-search-scope-policy.cjs +0 -49
- package/bin/codebase-search-runtime.cjs +0 -23
- package/bin/cognitive-action-checkpoint.cjs +0 -1104
- package/bin/cognitive-attention-delivery.cjs +0 -76
- package/bin/cognitive-attention-policy.cjs +0 -143
- package/bin/cognitive-attention-runtime.cjs +0 -91
- package/bin/cognitive-context-projection.cjs +0 -73
- package/bin/cognitive-cross-portal-acceptance.cjs +0 -443
- package/bin/cognitive-effective-view.cjs +0 -77
- package/bin/cognitive-focus-projection.cjs +0 -206
- package/bin/cognitive-focus-scope.cjs +0 -37
- package/bin/cognitive-goal-autostart-policy.cjs +0 -72
- package/bin/cognitive-goal-time-trigger-controller.cjs +0 -146
- package/bin/cognitive-memory-adapter.cjs +0 -282
- package/bin/cognitive-memory-command.cjs +0 -293
- package/bin/cognitive-memory-provider.cjs +0 -92
- package/bin/cognitive-salience-policy.cjs +0 -159
- package/bin/cognitive-state-store.cjs +0 -508
- package/bin/cognitive-turn-lifecycle.cjs +0 -624
- package/bin/cognitive-work-focus.cjs +0 -180
- package/bin/compaction-history-archive.cjs +0 -166
- package/bin/compaction-history-startup.cjs +0 -50
- package/bin/compaction-model-policy.cjs +0 -31
- package/bin/compaction-stage-policy.cjs +0 -21
- package/bin/compaction-transaction-policy.cjs +0 -122
- package/bin/config-write-dedup-policy.cjs +0 -27
- package/bin/context-budget-ledger.cjs +0 -31
- package/bin/context-doctor-policy.cjs +0 -70
- package/bin/context-insight-policy.cjs +0 -36
- package/bin/context-performance-policy.cjs +0 -19
- package/bin/context-pressure-policy.cjs +0 -20
- package/bin/cron-run-output.cjs +0 -45
- package/bin/cron-run-store.cjs +0 -145
- package/bin/curiosity-scout-policy.cjs +0 -49
- package/bin/default-model-output-budget-policy.cjs +0 -28
- package/bin/durable-task-resume-policy.cjs +0 -130
- package/bin/durable-task-resume-runtime.cjs +0 -117
- package/bin/durable-task-resume-store.cjs +0 -88
- package/bin/editable-tool-approval-policy.cjs +0 -540
- package/bin/editable-tool-approval-runtime.cjs +0 -99
- package/bin/effective-system-prompt-cache-policy.cjs +0 -33
- package/bin/error-memory-performance-policy.cjs +0 -113
- package/bin/file-observation-policy.cjs +0 -133
- package/bin/foreground-output-capture-policy.cjs +0 -41
- package/bin/generated-source-health.cjs +0 -142
- package/bin/glob-pattern-policy.cjs +0 -13
- package/bin/goal-completion-evidence-policy.cjs +0 -120
- package/bin/grep-output-limit-policy.cjs +0 -39
- package/bin/historical-media-projection-policy.cjs +0 -48
- package/bin/history-offload-pressure-policy.cjs +0 -33
- package/bin/html-to-research-markdown.cjs +0 -147
- package/bin/identity-context-policy.cjs +0 -764
- package/bin/identity-journal-policy.cjs +0 -107
- package/bin/input-draft-persistence.cjs +0 -77
- package/bin/king-tui-function-contract.json +0 -33
- package/bin/launcher-restart-policy.cjs +0 -150
- package/bin/live-response-repetition-guard.cjs +0 -196
- package/bin/llm-config-log-dedup-policy.cjs +0 -76
- package/bin/loop-event-record-policy.cjs +0 -174
- package/bin/managed-context-startup-policy.cjs +0 -27
- package/bin/media-activity-layout-policy.cjs +0 -34
- package/bin/media-auto-retrieval-policy.cjs +0 -90
- package/bin/media-result-policy.cjs +0 -59
- package/bin/micro-compaction-policy.cjs +0 -145
- package/bin/mistake-relevance-policy.cjs +0 -319
- package/bin/model-retry-progress-policy.cjs +0 -46
- package/bin/native-large-file-io.cjs +0 -42
- package/bin/native-runtime-cache.cjs +0 -76
- package/bin/natural-presence-policy.cjs +0 -28
- package/bin/noninteractive-shell-env-policy.cjs +0 -19
- package/bin/observer-hooks.cjs +0 -14
- package/bin/outbound-claim-provenance.cjs +0 -150
- package/bin/oversized-context-offload-policy.cjs +0 -86
- package/bin/pending-media-policy.cjs +0 -182
- package/bin/pending-token-estimate-policy.cjs +0 -41
- package/bin/personal-memory-consent-policy.cjs +0 -72
- package/bin/personal-memory-performance-policy.cjs +0 -12
- package/bin/personality-choice-policy.cjs +0 -101
- package/bin/personality-memory-adapter.cjs +0 -379
- package/bin/personality-mode.cjs +0 -46
- package/bin/personality-setup-policy.cjs +0 -197
- package/bin/proactive-compaction-policy.cjs +0 -25
- package/bin/profile-identity-resolution.cjs +0 -136
- package/bin/profile-runtime.cjs +0 -318
- package/bin/profile-tool-exclusion-policy.cjs +0 -37
- package/bin/programmatic-context-isolation.cjs +0 -25
- package/bin/programmatic-tool-runtime.mjs +0 -627
- package/bin/provider-idle-timeout-policy.cjs +0 -14
- package/bin/provider-model-refresh-deadline.cjs +0 -53
- package/bin/provider-model-refresh-policy.cjs +0 -107
- package/bin/rate-limit-recovery-policy.cjs +0 -47
- package/bin/read-batch-policy.cjs +0 -32
- package/bin/read-continuation-policy.cjs +0 -59
- package/bin/recurring-cron-history-policy.cjs +0 -124
- package/bin/relationship-continuity-policy.cjs +0 -143
- package/bin/relationship-curiosity-policy.cjs +0 -107
- package/bin/relationship-learning-policy.cjs +0 -168
- package/bin/release-artifact-freeze-policy.cjs +0 -30
- package/bin/reload-plugin-bootstrap.cjs +0 -18
- package/bin/reload-queue-policy.cjs +0 -38
- package/bin/repeated-assistant-response-policy.cjs +0 -232
- package/bin/repeated-injection-projection.cjs +0 -107
- package/bin/repeated-user-message-projection.cjs +0 -8
- package/bin/research-page-result.cjs +0 -74
- package/bin/retry-checkpoint-policy.cjs +0 -13
- package/bin/runtime-exit-ledger.cjs +0 -144
- package/bin/scoped-cron-run-policy.cjs +0 -358
- package/bin/session-checkpoint-policy.cjs +0 -25
- package/bin/session-compaction-policy.cjs +0 -84
- package/bin/session-replay-policy.cjs +0 -20
- package/bin/session-replay-window-policy.cjs +0 -40
- package/bin/session-resume-checkpoint.cjs +0 -254
- package/bin/session-scrollback-archive.cjs +0 -229
- package/bin/skill-activation-performance-policy.cjs +0 -69
- package/bin/skill-listing-performance-policy.cjs +0 -92
- package/bin/soul-organization-policy.cjs +0 -78
- package/bin/soul-preservation-policy.cjs +0 -20
- package/bin/startup-preferences.cjs +0 -131
- package/bin/streaming-flush-performance-policy.cjs +0 -28
- package/bin/structured-agent-swarm-output.cjs +0 -325
- package/bin/structured-subagent-output.cjs +0 -252
- package/bin/subagent-context-fork-policy.cjs +0 -155
- package/bin/subagent-max-tokens-handoff-policy.cjs +0 -69
- package/bin/subagent-parent-responsiveness.cjs +0 -19
- package/bin/subagent-skill-policy.cjs +0 -206
- package/bin/subagent-timeout-policy.cjs +0 -182
- package/bin/subagent-tool-policy.cjs +0 -60
- package/bin/subagent-usage-rollup-policy.cjs +0 -29
- package/bin/system-prompt-context-policy.cjs +0 -124
- package/bin/system-prompt-token-cache-policy.cjs +0 -60
- package/bin/telegram-addressed-focus.cjs +0 -55
- package/bin/telegram-addressed-priority.cjs +0 -12
- package/bin/telegram-approval-relay.cjs +0 -290
- package/bin/telegram-bot-priority.cjs +0 -17
- package/bin/telegram-console-status-policy.cjs +0 -174
- package/bin/telegram-context-projection-policy.cjs +0 -141
- package/bin/telegram-delivery-lifecycle.cjs +0 -125
- package/bin/telegram-direct-focus-policy.cjs +0 -273
- package/bin/telegram-mcp-compatibility.cjs +0 -49
- package/bin/telegram-media-delivery-policy.cjs +0 -42
- package/bin/telegram-private-conversation-policy.cjs +0 -185
- package/bin/telegram-queue-handoff-policy.cjs +0 -73
- package/bin/telegram-remote-status-policy.cjs +0 -120
- package/bin/telegram-session-queue-runtime.mjs +0 -306
- package/bin/telegram-text-chunk-policy.cjs +0 -63
- package/bin/telegram-truncated-reply-policy.cjs +0 -37
- package/bin/telegram-urgent-policy.cjs +0 -45
- package/bin/telemetry-spool-policy.cjs +0 -57
- package/bin/thinking-activity-status-policy.cjs +0 -132
- package/bin/thinking-only-guard.cjs +0 -80
- package/bin/todo-list-turn-policy.cjs +0 -131
- package/bin/tool-call-loop-policy.cjs +0 -51
- package/bin/tool-file-persistence.cjs +0 -141
- package/bin/tool-result-offload-policy.cjs +0 -359
- package/bin/tool-result-offload-telemetry.cjs +0 -12
- package/bin/tool-schema-token-cache-policy.cjs +0 -41
- package/bin/tool-stream-preview-policy.cjs +0 -9
- package/bin/tui-functional-contract.cjs +0 -55
- package/bin/turn-tool-performance-policy.cjs +0 -486
- package/bin/usage-cache-efficiency-policy.cjs +0 -26
- package/bin/user-home-path-policy.cjs +0 -13
- package/bin/user-message-offload-policy.cjs +0 -103
- package/bin/user-prompt-hook-origin-policy.cjs +0 -34
- package/bin/user-tool-record-policy.cjs +0 -7
- package/bin/validated-learning-insight-policy.cjs +0 -58
- package/bin/validated-learning-outcome-trace.cjs +0 -107
- package/bin/validated-learning-performance-policy.cjs +0 -53
- package/bin/validated-learning-signal.cjs +0 -463
- package/bin/windows-bash-dialect-policy.cjs +0 -25
- package/bin/windows-node-crash-dump.cjs +0 -110
- package/bin/write-continuation-policy.cjs +0 -69
- package/codebase-index/README.md +0 -82
- package/codebase-index/codebase_index.py +0 -470
- package/standard-skills/agent-browser/SKILL.md +0 -19
- package/standard-skills/agent-browser/references/runtime.md +0 -8
- package/standard-skills/blun-session-inspector/SKILL.md +0 -41
- package/standard-skills/blun-session-inspector/scripts/inspect-session.cjs +0 -437
- package/standard-skills/design-taste-frontend/SKILL.md +0 -1206
- package/standard-skills/full-output-enforcement/SKILL.md +0 -49
- package/standard-skills/high-end-visual-design/SKILL.md +0 -98
- package/standard-skills/image-to-code/SKILL.md +0 -1228
- package/standard-skills/industrial-brutalist-ui/SKILL.md +0 -92
- package/standard-skills/minimalist-ui/SKILL.md +0 -85
- package/standard-skills/motion-design-taste/SKILL.md +0 -74
- package/standard-skills/playwright-testing/SKILL.md +0 -19
- package/standard-skills/playwright-testing/references/runtime.md +0 -7
- package/standard-skills/premortem/SKILL.md +0 -148
- package/standard-skills/redesign-existing-projects/SKILL.md +0 -178
- package/standard-skills/research-evidence/SKILL.md +0 -39
- package/standard-skills/research-evidence/references/evidence-format.md +0 -104
- package/standard-skills/research-evidence/scripts/evidence-collection.cjs +0 -260
- package/standard-skills/research-evidence/scripts/score-report.cjs +0 -130
- package/standard-skills/screenshot-lesen/SKILL.md +0 -52
- package/standard-skills/stitch-design-taste/DESIGN.md +0 -121
- package/standard-skills/stitch-design-taste/SKILL.md +0 -184
- package/standard-skills/telegram-channel/SKILL.md +0 -18
- package/standard-skills/telegram-channel/references/runtime.md +0 -7
- package/standard-skills/venture-flywheel/SKILL.md +0 -32
- package/standard-skills/venture-flywheel/identity/project-identity.cjs +0 -146
- package/standard-skills/venture-flywheel/policy/capability-engine.cjs +0 -114
- package/standard-skills/venture-flywheel/policy/repository-trust.cjs +0 -229
- package/standard-skills/venture-flywheel/references/BEISPIELE-phase0.md +0 -146
- package/standard-skills/venture-flywheel/references/CAPABILITY-MAP.md +0 -34
- package/standard-skills/venture-flywheel/references/SPEC-phase0-identity-trust.md +0 -77
- package/standard-skills/venture-flywheel/references/SPEC-phase0-state-events.md +0 -93
- package/standard-skills/venture-flywheel/schemas/capability-decision.schema.json +0 -13
- package/standard-skills/venture-flywheel/schemas/execution-event.schema.json +0 -44
- package/standard-skills/venture-flywheel/schemas/project-identity.schema.json +0 -32
- package/standard-skills/venture-flywheel/schemas/repository-trust.schema.json +0 -57
- package/standard-skills/venture-flywheel/schemas/run-transition.schema.json +0 -59
- package/standard-skills/venture-flywheel/state/execution-event.cjs +0 -191
- package/standard-skills/venture-flywheel/state/task-state-machine.cjs +0 -190
- package/standard-skills/web-lesen/SKILL.md +0 -73
- package/standard-skills/web-lesen/scripts/crawl_public.py +0 -379
- package/standard-skills/windows-mcp/SKILL.md +0 -19
- package/standard-skills/windows-mcp/references/runtime.md +0 -9
- package/telegram-plugin/DELIVERY.md +0 -36
- package/telegram-plugin/bin/telegram-approval-relay.cjs +0 -290
- package/telegram-plugin/bin/telegram-console-status-policy.cjs +0 -175
- package/telegram-plugin/bin/telegram-delivery-lifecycle.cjs +0 -125
- package/telegram-plugin/bin/telegram-direct-reply-policy.cjs +0 -48
- package/telegram-plugin/bin/telegram-launcher-status-queue.cjs +0 -122
- package/telegram-plugin/bin/telegram-private-conversation-policy.cjs +0 -186
- package/telegram-plugin/bin/telegram-remote-status-policy.cjs +0 -121
- package/telegram-plugin/bin/telegram-reply-parts.cjs +0 -149
- package/telegram-plugin/bin/telegram-text-chunk-policy.cjs +0 -63
- package/telegram-plugin/bin/telegram-typing-keepalive.cjs +0 -89
- package/telegram-plugin/compat/mcp-server-fa511cd1.mjs +0 -73825
- /package/{bin → scripts}/fix-node-pty-perms.js +0 -0
package/standard-skills/translate-native/integrations/website_localization_native_reference_queue.py
ADDED
|
@@ -0,0 +1,1449 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Durable editorial queue for qualified-native benchmark references.
|
|
3
|
+
|
|
4
|
+
Only identifiers, lease state, stable error codes, and the accepted artifact
|
|
5
|
+
digest are stored here. Source and target prose remain in the separately
|
|
6
|
+
bound work order and native-reference artifact.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import importlib.util
|
|
13
|
+
import json
|
|
14
|
+
import math
|
|
15
|
+
import re
|
|
16
|
+
import secrets
|
|
17
|
+
import sqlite3
|
|
18
|
+
import sys
|
|
19
|
+
import unicodedata
|
|
20
|
+
from contextlib import contextmanager
|
|
21
|
+
from dataclasses import asdict, dataclass
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any, Iterator
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
SCHEMA = "blun.website-localization-native-reference-queue.v1"
|
|
27
|
+
HEALTH_SCHEMA = "blun.website-localization-native-reference-queue-health.v1"
|
|
28
|
+
STATUSES = ("pending", "leased", "retry_wait", "succeeded", "failed")
|
|
29
|
+
MAX_ATTEMPTS = 20
|
|
30
|
+
MAX_LEASE_SECONDS = 86_400.0
|
|
31
|
+
MAX_STALE_SECONDS = 31_536_000.0
|
|
32
|
+
ERROR_CODE = re.compile(r"^[a-z][a-z0-9_.-]{0,127}$")
|
|
33
|
+
SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
|
34
|
+
TABLE_COLUMNS = (
|
|
35
|
+
"work_id", "campaign_id", "target_locale", "suite_case_key", "status",
|
|
36
|
+
"attempts", "max_attempts", "next_attempt_at", "lease_owner",
|
|
37
|
+
"lease_token", "lease_expires_at", "last_error_code",
|
|
38
|
+
"last_error_detail_hash", "artifact_sha256", "created_at", "updated_at",
|
|
39
|
+
)
|
|
40
|
+
CLAIM_REQUEST_COLUMNS = (
|
|
41
|
+
"request_id", "campaign_id", "editor_id", "target_locale", "work_id",
|
|
42
|
+
"attempt", "lease_token", "lease_expires_at", "created_at",
|
|
43
|
+
)
|
|
44
|
+
HTTP_REQUEST_COLUMNS = (
|
|
45
|
+
"request_id", "campaign_id", "editor_id", "target_locale", "operation",
|
|
46
|
+
"request_sha256", "work_id", "attempt", "lease_token",
|
|
47
|
+
"lease_expires_at", "status", "result_json", "result_sha256",
|
|
48
|
+
"created_at", "updated_at",
|
|
49
|
+
)
|
|
50
|
+
HTTP_OPERATIONS = ("renew", "submit")
|
|
51
|
+
HTTP_REQUEST_STATUSES = ("processing", "completed", "abandoned")
|
|
52
|
+
OUTCOME_FIELDS = (
|
|
53
|
+
"work_id", "target_locale", "suite_case_key", "status", "attempt",
|
|
54
|
+
"max_attempts", "next_attempt_at", "error_code", "error_detail_hash",
|
|
55
|
+
"artifact_sha256",
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _load_module(name: str, path: Path):
|
|
60
|
+
spec = importlib.util.spec_from_file_location(name, path)
|
|
61
|
+
if spec is None or spec.loader is None:
|
|
62
|
+
raise RuntimeError(f"cannot load native-reference queue dependency: {path.name}")
|
|
63
|
+
module = importlib.util.module_from_spec(spec)
|
|
64
|
+
sys.modules[spec.name] = module
|
|
65
|
+
spec.loader.exec_module(module)
|
|
66
|
+
return module
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
_ROOT = Path(__file__).resolve().parents[1]
|
|
70
|
+
_CAMPAIGN = _load_module(
|
|
71
|
+
"blun_website_localization_native_reference_queue_campaign",
|
|
72
|
+
_ROOT / "integrations" / "website_localization_benchmark_campaign.py",
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class NativeReferenceQueueBlocked(RuntimeError):
|
|
77
|
+
"""Content-free queue failure safe for operational reporting."""
|
|
78
|
+
|
|
79
|
+
def __init__(self, code: str):
|
|
80
|
+
if not isinstance(code, str) or ERROR_CODE.fullmatch(code) is None:
|
|
81
|
+
raise ValueError("native-reference queue error code is invalid")
|
|
82
|
+
super().__init__(code)
|
|
83
|
+
self.code = code
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True)
|
|
87
|
+
class ClaimedNativeReference:
|
|
88
|
+
work_id: str
|
|
89
|
+
campaign_id: str
|
|
90
|
+
target_locale: str
|
|
91
|
+
suite_case_key: str
|
|
92
|
+
job_payload: dict[str, Any]
|
|
93
|
+
attempt: int
|
|
94
|
+
max_attempts: int
|
|
95
|
+
lease_owner: str
|
|
96
|
+
lease_token: str
|
|
97
|
+
lease_expires_at: float
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass(frozen=True)
|
|
101
|
+
class NativeReferenceQueueOutcome:
|
|
102
|
+
work_id: str
|
|
103
|
+
target_locale: str
|
|
104
|
+
suite_case_key: str
|
|
105
|
+
status: str
|
|
106
|
+
attempt: int
|
|
107
|
+
max_attempts: int
|
|
108
|
+
next_attempt_at: float
|
|
109
|
+
error_code: str | None
|
|
110
|
+
error_detail_hash: str | None
|
|
111
|
+
artifact_sha256: str | None
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@dataclass(frozen=True)
|
|
115
|
+
class NativeReferenceQueueHealth:
|
|
116
|
+
campaign_id: str
|
|
117
|
+
status: str
|
|
118
|
+
reasons: tuple[str, ...]
|
|
119
|
+
counts: tuple[tuple[str, int], ...]
|
|
120
|
+
work_count: int
|
|
121
|
+
last_progress_at: float | None
|
|
122
|
+
|
|
123
|
+
def as_payload(self) -> dict[str, Any]:
|
|
124
|
+
return {
|
|
125
|
+
"schema": HEALTH_SCHEMA,
|
|
126
|
+
"campaign_id": self.campaign_id,
|
|
127
|
+
"status": self.status,
|
|
128
|
+
"reasons": list(self.reasons),
|
|
129
|
+
"counts": dict(self.counts),
|
|
130
|
+
"work_count": self.work_count,
|
|
131
|
+
"last_progress_at": self.last_progress_at,
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _timestamp(value: Any) -> float:
|
|
136
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
137
|
+
raise NativeReferenceQueueBlocked("native_reference.queue.time_invalid")
|
|
138
|
+
value = float(value)
|
|
139
|
+
if not math.isfinite(value) or value < 0:
|
|
140
|
+
raise NativeReferenceQueueBlocked("native_reference.queue.time_invalid")
|
|
141
|
+
return value
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _duration(value: Any, *, allow_zero: bool = False) -> float:
|
|
145
|
+
value = _timestamp(value)
|
|
146
|
+
if (value < 0 if allow_zero else value <= 0) or value > MAX_LEASE_SECONDS:
|
|
147
|
+
raise NativeReferenceQueueBlocked("native_reference.queue.duration_invalid")
|
|
148
|
+
return value
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _identifier(value: Any) -> str:
|
|
152
|
+
if (
|
|
153
|
+
not isinstance(value, str) or not value or value != value.strip()
|
|
154
|
+
or len(value) > 256 or "\x00" in value
|
|
155
|
+
or not unicodedata.is_normalized("NFC", value)
|
|
156
|
+
):
|
|
157
|
+
raise NativeReferenceQueueBlocked("native_reference.queue.identity_invalid")
|
|
158
|
+
return value
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@contextmanager
|
|
162
|
+
def _transaction(connection: sqlite3.Connection) -> Iterator[None]:
|
|
163
|
+
if connection.in_transaction:
|
|
164
|
+
raise NativeReferenceQueueBlocked(
|
|
165
|
+
"native_reference.queue.external_transaction",
|
|
166
|
+
)
|
|
167
|
+
try:
|
|
168
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
169
|
+
yield
|
|
170
|
+
except Exception:
|
|
171
|
+
connection.rollback()
|
|
172
|
+
raise
|
|
173
|
+
else:
|
|
174
|
+
connection.commit()
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
class NativeReferenceWorkQueue:
|
|
178
|
+
"""Lease exact campaign jobs to independent native editorial workers."""
|
|
179
|
+
|
|
180
|
+
def __init__(self, campaign_store: Any):
|
|
181
|
+
connection = getattr(campaign_store, "connection", None)
|
|
182
|
+
if (
|
|
183
|
+
not isinstance(connection, sqlite3.Connection)
|
|
184
|
+
or not callable(getattr(campaign_store, "_verify_binding_locked", None))
|
|
185
|
+
):
|
|
186
|
+
raise NativeReferenceQueueBlocked(
|
|
187
|
+
"native_reference.queue.campaign_store_invalid",
|
|
188
|
+
)
|
|
189
|
+
self.campaign_store = campaign_store
|
|
190
|
+
self.connection = connection
|
|
191
|
+
self.connection.row_factory = sqlite3.Row
|
|
192
|
+
with _transaction(self.connection):
|
|
193
|
+
self.connection.execute(f"""
|
|
194
|
+
CREATE TABLE IF NOT EXISTS benchmark_native_reference_queue (
|
|
195
|
+
work_id TEXT PRIMARY KEY,
|
|
196
|
+
campaign_id TEXT NOT NULL,
|
|
197
|
+
target_locale TEXT NOT NULL,
|
|
198
|
+
suite_case_key TEXT NOT NULL,
|
|
199
|
+
status TEXT NOT NULL CHECK (status IN {STATUSES}),
|
|
200
|
+
attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0),
|
|
201
|
+
max_attempts INTEGER NOT NULL CHECK (
|
|
202
|
+
max_attempts BETWEEN 1 AND {MAX_ATTEMPTS}
|
|
203
|
+
),
|
|
204
|
+
next_attempt_at REAL NOT NULL,
|
|
205
|
+
lease_owner TEXT,
|
|
206
|
+
lease_token TEXT,
|
|
207
|
+
lease_expires_at REAL,
|
|
208
|
+
last_error_code TEXT,
|
|
209
|
+
last_error_detail_hash TEXT,
|
|
210
|
+
artifact_sha256 TEXT,
|
|
211
|
+
created_at REAL NOT NULL,
|
|
212
|
+
updated_at REAL NOT NULL,
|
|
213
|
+
UNIQUE (campaign_id, target_locale, suite_case_key),
|
|
214
|
+
FOREIGN KEY (work_id) REFERENCES benchmark_campaign_work (work_id),
|
|
215
|
+
FOREIGN KEY (campaign_id) REFERENCES benchmark_campaigns (campaign_id)
|
|
216
|
+
)
|
|
217
|
+
""")
|
|
218
|
+
self.connection.execute("""
|
|
219
|
+
CREATE INDEX IF NOT EXISTS benchmark_native_reference_queue_ready
|
|
220
|
+
ON benchmark_native_reference_queue
|
|
221
|
+
(campaign_id, status, next_attempt_at, target_locale, suite_case_key)
|
|
222
|
+
""")
|
|
223
|
+
self.connection.execute("""
|
|
224
|
+
CREATE TABLE IF NOT EXISTS benchmark_native_reference_claim_requests (
|
|
225
|
+
request_id TEXT PRIMARY KEY,
|
|
226
|
+
campaign_id TEXT NOT NULL,
|
|
227
|
+
editor_id TEXT NOT NULL,
|
|
228
|
+
target_locale TEXT NOT NULL,
|
|
229
|
+
work_id TEXT NOT NULL,
|
|
230
|
+
attempt INTEGER NOT NULL CHECK (attempt > 0),
|
|
231
|
+
lease_token TEXT NOT NULL,
|
|
232
|
+
lease_expires_at REAL NOT NULL,
|
|
233
|
+
created_at REAL NOT NULL,
|
|
234
|
+
FOREIGN KEY (work_id) REFERENCES benchmark_native_reference_queue(work_id),
|
|
235
|
+
FOREIGN KEY (campaign_id) REFERENCES benchmark_campaigns(campaign_id)
|
|
236
|
+
)
|
|
237
|
+
""")
|
|
238
|
+
self.connection.execute(f"""
|
|
239
|
+
CREATE TABLE IF NOT EXISTS benchmark_native_reference_http_requests (
|
|
240
|
+
request_id TEXT PRIMARY KEY,
|
|
241
|
+
campaign_id TEXT NOT NULL,
|
|
242
|
+
editor_id TEXT NOT NULL,
|
|
243
|
+
target_locale TEXT NOT NULL,
|
|
244
|
+
operation TEXT NOT NULL CHECK (operation IN {HTTP_OPERATIONS}),
|
|
245
|
+
request_sha256 TEXT NOT NULL,
|
|
246
|
+
work_id TEXT NOT NULL,
|
|
247
|
+
attempt INTEGER NOT NULL CHECK (attempt > 0),
|
|
248
|
+
lease_token TEXT NOT NULL,
|
|
249
|
+
lease_expires_at REAL NOT NULL,
|
|
250
|
+
status TEXT NOT NULL CHECK (status IN {HTTP_REQUEST_STATUSES}),
|
|
251
|
+
result_json TEXT,
|
|
252
|
+
result_sha256 TEXT,
|
|
253
|
+
created_at REAL NOT NULL,
|
|
254
|
+
updated_at REAL NOT NULL,
|
|
255
|
+
FOREIGN KEY (work_id) REFERENCES benchmark_native_reference_queue(work_id),
|
|
256
|
+
FOREIGN KEY (campaign_id) REFERENCES benchmark_campaigns(campaign_id)
|
|
257
|
+
)
|
|
258
|
+
""")
|
|
259
|
+
self.connection.execute("""
|
|
260
|
+
CREATE INDEX IF NOT EXISTS benchmark_native_reference_http_work
|
|
261
|
+
ON benchmark_native_reference_http_requests
|
|
262
|
+
(campaign_id, work_id, attempt, operation)
|
|
263
|
+
""")
|
|
264
|
+
self._verify_schema()
|
|
265
|
+
|
|
266
|
+
def _verify_schema(self) -> None:
|
|
267
|
+
columns = tuple(
|
|
268
|
+
row[1] for row in self.connection.execute(
|
|
269
|
+
"PRAGMA table_info(benchmark_native_reference_queue)"
|
|
270
|
+
)
|
|
271
|
+
)
|
|
272
|
+
if columns != TABLE_COLUMNS:
|
|
273
|
+
raise NativeReferenceQueueBlocked(
|
|
274
|
+
"native_reference.queue.schema_unsupported",
|
|
275
|
+
)
|
|
276
|
+
request_columns = tuple(
|
|
277
|
+
row[1] for row in self.connection.execute(
|
|
278
|
+
"PRAGMA table_info(benchmark_native_reference_claim_requests)"
|
|
279
|
+
)
|
|
280
|
+
)
|
|
281
|
+
if request_columns != CLAIM_REQUEST_COLUMNS:
|
|
282
|
+
raise NativeReferenceQueueBlocked(
|
|
283
|
+
"native_reference.queue.schema_unsupported",
|
|
284
|
+
)
|
|
285
|
+
http_columns = tuple(
|
|
286
|
+
row[1] for row in self.connection.execute(
|
|
287
|
+
"PRAGMA table_info(benchmark_native_reference_http_requests)"
|
|
288
|
+
)
|
|
289
|
+
)
|
|
290
|
+
if http_columns != HTTP_REQUEST_COLUMNS:
|
|
291
|
+
raise NativeReferenceQueueBlocked(
|
|
292
|
+
"native_reference.queue.schema_unsupported",
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
def ensure(
|
|
296
|
+
self, policy: Any, campaign_id: str, *, max_attempts: Any, now: Any,
|
|
297
|
+
) -> None:
|
|
298
|
+
if (
|
|
299
|
+
isinstance(max_attempts, bool) or not isinstance(max_attempts, int)
|
|
300
|
+
or not 1 <= max_attempts <= MAX_ATTEMPTS
|
|
301
|
+
):
|
|
302
|
+
raise NativeReferenceQueueBlocked(
|
|
303
|
+
"native_reference.queue.attempts_invalid",
|
|
304
|
+
)
|
|
305
|
+
now = _timestamp(now)
|
|
306
|
+
try:
|
|
307
|
+
policy = _CAMPAIGN._BENCHMARK._validate_policy(policy)
|
|
308
|
+
_CAMPAIGN._assert_policy_current(policy, now)
|
|
309
|
+
expected = _CAMPAIGN._expected_work(policy)
|
|
310
|
+
with _transaction(self.connection):
|
|
311
|
+
self.campaign_store._verify_binding_locked(policy, campaign_id)
|
|
312
|
+
self.connection.executemany("""
|
|
313
|
+
INSERT OR IGNORE INTO benchmark_native_reference_queue (
|
|
314
|
+
work_id, campaign_id, target_locale, suite_case_key,
|
|
315
|
+
status, attempts, max_attempts, next_attempt_at,
|
|
316
|
+
created_at, updated_at
|
|
317
|
+
) VALUES (?, ?, ?, ?, 'pending', 0, ?, ?, ?, ?)
|
|
318
|
+
""", (
|
|
319
|
+
(work_id, campaign_id, locale, case_key, max_attempts, now, now, now)
|
|
320
|
+
for work_id, locale, case_key in expected
|
|
321
|
+
))
|
|
322
|
+
self._verify_binding_locked(policy, campaign_id)
|
|
323
|
+
configured = self.connection.execute("""
|
|
324
|
+
SELECT MIN(max_attempts), MAX(max_attempts)
|
|
325
|
+
FROM benchmark_native_reference_queue WHERE campaign_id = ?
|
|
326
|
+
""", (campaign_id,)).fetchone()
|
|
327
|
+
if tuple(configured) != (max_attempts, max_attempts):
|
|
328
|
+
raise NativeReferenceQueueBlocked(
|
|
329
|
+
"native_reference.queue.attempts_mismatch",
|
|
330
|
+
)
|
|
331
|
+
except NativeReferenceQueueBlocked:
|
|
332
|
+
raise
|
|
333
|
+
except Exception:
|
|
334
|
+
raise NativeReferenceQueueBlocked(
|
|
335
|
+
"native_reference.queue.binding_invalid",
|
|
336
|
+
) from None
|
|
337
|
+
|
|
338
|
+
def _verify_binding_locked(self, policy: Any, campaign_id: str) -> None:
|
|
339
|
+
self.campaign_store._verify_binding_locked(policy, campaign_id)
|
|
340
|
+
expected = set(_CAMPAIGN._expected_work(policy))
|
|
341
|
+
rows = self.connection.execute("""
|
|
342
|
+
SELECT *
|
|
343
|
+
FROM benchmark_native_reference_queue WHERE campaign_id = ?
|
|
344
|
+
""", (campaign_id,)).fetchall()
|
|
345
|
+
campaign_attempts = {
|
|
346
|
+
row["work_id"]: row["max_attempts"]
|
|
347
|
+
for row in self.connection.execute("""
|
|
348
|
+
SELECT work_id, max_attempts FROM benchmark_campaign_work
|
|
349
|
+
WHERE campaign_id = ?
|
|
350
|
+
""", (campaign_id,))
|
|
351
|
+
}
|
|
352
|
+
observed = set()
|
|
353
|
+
for row in rows:
|
|
354
|
+
self._validate_row(row)
|
|
355
|
+
if campaign_attempts.get(row["work_id"]) != row["max_attempts"]:
|
|
356
|
+
raise NativeReferenceQueueBlocked(
|
|
357
|
+
"native_reference.queue.state_invalid",
|
|
358
|
+
)
|
|
359
|
+
observed.add((
|
|
360
|
+
row["work_id"], row["target_locale"], row["suite_case_key"],
|
|
361
|
+
))
|
|
362
|
+
if observed != expected:
|
|
363
|
+
raise NativeReferenceQueueBlocked(
|
|
364
|
+
"native_reference.queue.state_invalid",
|
|
365
|
+
)
|
|
366
|
+
requests = self.connection.execute("""
|
|
367
|
+
SELECT * FROM benchmark_native_reference_claim_requests
|
|
368
|
+
WHERE campaign_id = ?
|
|
369
|
+
""", (campaign_id,)).fetchall()
|
|
370
|
+
by_work_id = {row["work_id"]: row for row in rows}
|
|
371
|
+
for request in requests:
|
|
372
|
+
try:
|
|
373
|
+
queued = by_work_id.get(request["work_id"])
|
|
374
|
+
if (
|
|
375
|
+
tuple(request.keys()) != CLAIM_REQUEST_COLUMNS
|
|
376
|
+
or queued is None
|
|
377
|
+
or request["campaign_id"] != campaign_id
|
|
378
|
+
or request["target_locale"] != queued["target_locale"]
|
|
379
|
+
or request["target_locale"] not in policy.required_locales
|
|
380
|
+
or isinstance(request["attempt"], bool)
|
|
381
|
+
or not isinstance(request["attempt"], int)
|
|
382
|
+
or not 1 <= request["attempt"] <= queued["max_attempts"]
|
|
383
|
+
):
|
|
384
|
+
raise ValueError
|
|
385
|
+
_identifier(request["request_id"])
|
|
386
|
+
_identifier(request["editor_id"])
|
|
387
|
+
_identifier(request["lease_token"])
|
|
388
|
+
expires = _timestamp(request["lease_expires_at"])
|
|
389
|
+
created = _timestamp(request["created_at"])
|
|
390
|
+
if expires <= created:
|
|
391
|
+
raise ValueError
|
|
392
|
+
if (
|
|
393
|
+
queued["status"] == "leased"
|
|
394
|
+
and queued["attempts"] == request["attempt"]
|
|
395
|
+
and queued["lease_owner"] == request["editor_id"]
|
|
396
|
+
) and (
|
|
397
|
+
queued["lease_token"] != request["lease_token"]
|
|
398
|
+
or queued["lease_expires_at"] != expires
|
|
399
|
+
):
|
|
400
|
+
raise ValueError
|
|
401
|
+
except NativeReferenceQueueBlocked:
|
|
402
|
+
raise
|
|
403
|
+
except Exception:
|
|
404
|
+
raise NativeReferenceQueueBlocked(
|
|
405
|
+
"native_reference.queue.state_invalid",
|
|
406
|
+
) from None
|
|
407
|
+
http_requests = self.connection.execute("""
|
|
408
|
+
SELECT * FROM benchmark_native_reference_http_requests
|
|
409
|
+
WHERE campaign_id = ?
|
|
410
|
+
""", (campaign_id,)).fetchall()
|
|
411
|
+
for request in http_requests:
|
|
412
|
+
queued = by_work_id.get(request["work_id"])
|
|
413
|
+
self._validate_http_request_row(request, queued, policy)
|
|
414
|
+
|
|
415
|
+
@staticmethod
|
|
416
|
+
def _canonical_json(value: Any) -> str:
|
|
417
|
+
try:
|
|
418
|
+
return json.dumps(
|
|
419
|
+
value,
|
|
420
|
+
ensure_ascii=False,
|
|
421
|
+
allow_nan=False,
|
|
422
|
+
sort_keys=True,
|
|
423
|
+
separators=(",", ":"),
|
|
424
|
+
)
|
|
425
|
+
except (TypeError, ValueError, UnicodeEncodeError, RecursionError):
|
|
426
|
+
raise NativeReferenceQueueBlocked(
|
|
427
|
+
"native_reference.queue.http_request_invalid",
|
|
428
|
+
) from None
|
|
429
|
+
|
|
430
|
+
@staticmethod
|
|
431
|
+
def _validate_http_result(operation: str, value: Any) -> dict[str, Any]:
|
|
432
|
+
try:
|
|
433
|
+
if not isinstance(value, dict):
|
|
434
|
+
raise ValueError
|
|
435
|
+
if operation == "renew":
|
|
436
|
+
if set(value) != {"lease_expires_at"}:
|
|
437
|
+
raise ValueError
|
|
438
|
+
_timestamp(value["lease_expires_at"])
|
|
439
|
+
elif operation == "submit":
|
|
440
|
+
if set(value) != set(OUTCOME_FIELDS):
|
|
441
|
+
raise ValueError
|
|
442
|
+
_identifier(value["work_id"])
|
|
443
|
+
_identifier(value["target_locale"])
|
|
444
|
+
_identifier(value["suite_case_key"])
|
|
445
|
+
if value["status"] not in {"retry_wait", "succeeded", "failed"}:
|
|
446
|
+
raise ValueError
|
|
447
|
+
attempt = value["attempt"]
|
|
448
|
+
maximum = value["max_attempts"]
|
|
449
|
+
if (
|
|
450
|
+
isinstance(attempt, bool) or not isinstance(attempt, int)
|
|
451
|
+
or isinstance(maximum, bool) or not isinstance(maximum, int)
|
|
452
|
+
or not 1 <= attempt <= maximum <= MAX_ATTEMPTS
|
|
453
|
+
):
|
|
454
|
+
raise ValueError
|
|
455
|
+
_timestamp(value["next_attempt_at"])
|
|
456
|
+
for field in ("error_code", "error_detail_hash", "artifact_sha256"):
|
|
457
|
+
item = value[field]
|
|
458
|
+
if item is not None and not isinstance(item, str):
|
|
459
|
+
raise ValueError
|
|
460
|
+
if value["error_code"] is not None and ERROR_CODE.fullmatch(
|
|
461
|
+
value["error_code"],
|
|
462
|
+
) is None:
|
|
463
|
+
raise ValueError
|
|
464
|
+
for field in ("error_detail_hash", "artifact_sha256"):
|
|
465
|
+
if value[field] is not None and SHA256.fullmatch(value[field]) is None:
|
|
466
|
+
raise ValueError
|
|
467
|
+
if value["status"] == "succeeded":
|
|
468
|
+
if (
|
|
469
|
+
value["error_code"] is not None
|
|
470
|
+
or value["error_detail_hash"] is not None
|
|
471
|
+
or value["artifact_sha256"] is None
|
|
472
|
+
):
|
|
473
|
+
raise ValueError
|
|
474
|
+
elif (
|
|
475
|
+
value["error_code"] is None
|
|
476
|
+
or value["artifact_sha256"] is not None
|
|
477
|
+
):
|
|
478
|
+
raise ValueError
|
|
479
|
+
else:
|
|
480
|
+
raise ValueError
|
|
481
|
+
except NativeReferenceQueueBlocked:
|
|
482
|
+
raise
|
|
483
|
+
except Exception:
|
|
484
|
+
raise NativeReferenceQueueBlocked(
|
|
485
|
+
"native_reference.queue.http_request_invalid",
|
|
486
|
+
) from None
|
|
487
|
+
return value
|
|
488
|
+
|
|
489
|
+
@classmethod
|
|
490
|
+
def _validate_http_request_row(
|
|
491
|
+
cls, request: sqlite3.Row, queued: sqlite3.Row | None, policy: Any,
|
|
492
|
+
) -> None:
|
|
493
|
+
try:
|
|
494
|
+
if (
|
|
495
|
+
tuple(request.keys()) != HTTP_REQUEST_COLUMNS
|
|
496
|
+
or queued is None
|
|
497
|
+
or request["campaign_id"] != queued["campaign_id"]
|
|
498
|
+
or request["target_locale"] != queued["target_locale"]
|
|
499
|
+
or request["target_locale"] not in policy.required_locales
|
|
500
|
+
or request["operation"] not in HTTP_OPERATIONS
|
|
501
|
+
or request["status"] not in HTTP_REQUEST_STATUSES
|
|
502
|
+
or not isinstance(request["attempt"], int)
|
|
503
|
+
or isinstance(request["attempt"], bool)
|
|
504
|
+
or not 1 <= request["attempt"] <= queued["max_attempts"]
|
|
505
|
+
):
|
|
506
|
+
raise ValueError
|
|
507
|
+
for field in ("request_id", "editor_id", "lease_token"):
|
|
508
|
+
_identifier(request[field])
|
|
509
|
+
if SHA256.fullmatch(request["request_sha256"]) is None:
|
|
510
|
+
raise ValueError
|
|
511
|
+
expires = _timestamp(request["lease_expires_at"])
|
|
512
|
+
created = _timestamp(request["created_at"])
|
|
513
|
+
updated = _timestamp(request["updated_at"])
|
|
514
|
+
if expires <= created or not created <= updated:
|
|
515
|
+
raise ValueError
|
|
516
|
+
result_json = request["result_json"]
|
|
517
|
+
result_sha256 = request["result_sha256"]
|
|
518
|
+
if request["status"] == "processing":
|
|
519
|
+
if result_json is not None or result_sha256 is not None:
|
|
520
|
+
raise ValueError
|
|
521
|
+
if (
|
|
522
|
+
queued["status"] != "leased"
|
|
523
|
+
or queued["attempts"] != request["attempt"]
|
|
524
|
+
or queued["lease_owner"] != request["editor_id"]
|
|
525
|
+
or queued["lease_token"] != request["lease_token"]
|
|
526
|
+
or queued["lease_expires_at"] != expires
|
|
527
|
+
):
|
|
528
|
+
raise ValueError
|
|
529
|
+
elif request["status"] == "completed":
|
|
530
|
+
if (
|
|
531
|
+
not isinstance(result_json, str)
|
|
532
|
+
or not isinstance(result_sha256, str)
|
|
533
|
+
or SHA256.fullmatch(result_sha256) is None
|
|
534
|
+
or hashlib.sha256(result_json.encode("utf-8")).hexdigest()
|
|
535
|
+
!= result_sha256
|
|
536
|
+
):
|
|
537
|
+
raise ValueError
|
|
538
|
+
result = json.loads(result_json)
|
|
539
|
+
if cls._canonical_json(result) != result_json:
|
|
540
|
+
raise ValueError
|
|
541
|
+
cls._validate_http_result(request["operation"], result)
|
|
542
|
+
if request["operation"] == "submit" and (
|
|
543
|
+
result["work_id"] != request["work_id"]
|
|
544
|
+
or result["target_locale"] != request["target_locale"]
|
|
545
|
+
or result["suite_case_key"] != queued["suite_case_key"]
|
|
546
|
+
or result["attempt"] != request["attempt"]
|
|
547
|
+
or result["max_attempts"] != queued["max_attempts"]
|
|
548
|
+
):
|
|
549
|
+
raise ValueError
|
|
550
|
+
elif result_json is not None or result_sha256 is not None:
|
|
551
|
+
raise ValueError
|
|
552
|
+
except NativeReferenceQueueBlocked:
|
|
553
|
+
raise
|
|
554
|
+
except Exception:
|
|
555
|
+
raise NativeReferenceQueueBlocked(
|
|
556
|
+
"native_reference.queue.state_invalid",
|
|
557
|
+
) from None
|
|
558
|
+
|
|
559
|
+
@staticmethod
|
|
560
|
+
def _validate_row(row: sqlite3.Row, *, now: float | None = None) -> None:
|
|
561
|
+
try:
|
|
562
|
+
if tuple(row.keys()) != TABLE_COLUMNS or row["status"] not in STATUSES:
|
|
563
|
+
raise ValueError
|
|
564
|
+
attempts = row["attempts"]
|
|
565
|
+
maximum = row["max_attempts"]
|
|
566
|
+
if (
|
|
567
|
+
isinstance(attempts, bool) or not isinstance(attempts, int)
|
|
568
|
+
or isinstance(maximum, bool) or not isinstance(maximum, int)
|
|
569
|
+
or not 0 <= attempts <= maximum <= MAX_ATTEMPTS
|
|
570
|
+
):
|
|
571
|
+
raise ValueError
|
|
572
|
+
created = _timestamp(row["created_at"])
|
|
573
|
+
updated = _timestamp(row["updated_at"])
|
|
574
|
+
_timestamp(row["next_attempt_at"])
|
|
575
|
+
if created > updated or now is not None and updated > now:
|
|
576
|
+
raise ValueError
|
|
577
|
+
lease_values = (
|
|
578
|
+
row["lease_owner"], row["lease_token"], row["lease_expires_at"],
|
|
579
|
+
)
|
|
580
|
+
if row["status"] == "leased":
|
|
581
|
+
_identifier(row["lease_owner"])
|
|
582
|
+
_identifier(row["lease_token"])
|
|
583
|
+
_timestamp(row["lease_expires_at"])
|
|
584
|
+
if attempts < 1:
|
|
585
|
+
raise ValueError
|
|
586
|
+
elif any(value is not None for value in lease_values):
|
|
587
|
+
raise ValueError
|
|
588
|
+
code = row["last_error_code"]
|
|
589
|
+
detail = row["last_error_detail_hash"]
|
|
590
|
+
artifact = row["artifact_sha256"]
|
|
591
|
+
if code is not None and (
|
|
592
|
+
not isinstance(code, str) or ERROR_CODE.fullmatch(code) is None
|
|
593
|
+
):
|
|
594
|
+
raise ValueError
|
|
595
|
+
if detail is not None and (
|
|
596
|
+
not isinstance(detail, str) or SHA256.fullmatch(detail) is None
|
|
597
|
+
):
|
|
598
|
+
raise ValueError
|
|
599
|
+
if row["status"] == "pending" and (
|
|
600
|
+
attempts != 0 or code is not None or detail is not None
|
|
601
|
+
):
|
|
602
|
+
raise ValueError
|
|
603
|
+
if row["status"] in {"leased", "succeeded"} and (
|
|
604
|
+
code is not None or detail is not None
|
|
605
|
+
):
|
|
606
|
+
raise ValueError
|
|
607
|
+
if row["status"] in {"retry_wait", "failed"} and (
|
|
608
|
+
attempts < 1 or code is None
|
|
609
|
+
):
|
|
610
|
+
raise ValueError
|
|
611
|
+
if row["status"] == "retry_wait" and attempts >= maximum:
|
|
612
|
+
raise ValueError
|
|
613
|
+
if row["status"] == "succeeded":
|
|
614
|
+
if (
|
|
615
|
+
attempts < 1 or not isinstance(artifact, str)
|
|
616
|
+
or SHA256.fullmatch(artifact) is None
|
|
617
|
+
):
|
|
618
|
+
raise ValueError
|
|
619
|
+
elif artifact is not None:
|
|
620
|
+
raise ValueError
|
|
621
|
+
except NativeReferenceQueueBlocked:
|
|
622
|
+
raise
|
|
623
|
+
except Exception:
|
|
624
|
+
raise NativeReferenceQueueBlocked(
|
|
625
|
+
"native_reference.queue.state_invalid",
|
|
626
|
+
) from None
|
|
627
|
+
|
|
628
|
+
def claim(
|
|
629
|
+
self, policy: Any, campaign_id: str, worker_id: Any, *,
|
|
630
|
+
now: Any, lease_seconds: Any = 3600, target_locale: Any = None,
|
|
631
|
+
request_id: Any = None,
|
|
632
|
+
) -> ClaimedNativeReference | None:
|
|
633
|
+
worker_id = _identifier(worker_id)
|
|
634
|
+
if target_locale is not None:
|
|
635
|
+
target_locale = _identifier(target_locale)
|
|
636
|
+
if request_id is not None:
|
|
637
|
+
request_id = _identifier(request_id)
|
|
638
|
+
now = _timestamp(now)
|
|
639
|
+
lease_seconds = _duration(lease_seconds)
|
|
640
|
+
try:
|
|
641
|
+
policy = _CAMPAIGN._BENCHMARK._validate_policy(policy)
|
|
642
|
+
_CAMPAIGN._assert_policy_current(policy, now)
|
|
643
|
+
except Exception:
|
|
644
|
+
raise NativeReferenceQueueBlocked(
|
|
645
|
+
"native_reference.queue.policy_invalid",
|
|
646
|
+
) from None
|
|
647
|
+
with _transaction(self.connection):
|
|
648
|
+
self._verify_binding_locked(policy, campaign_id)
|
|
649
|
+
if target_locale is not None and target_locale not in policy.required_locales:
|
|
650
|
+
raise NativeReferenceQueueBlocked(
|
|
651
|
+
"native_reference.queue.locale_not_allowed",
|
|
652
|
+
)
|
|
653
|
+
if request_id is not None:
|
|
654
|
+
mutation = self.connection.execute("""
|
|
655
|
+
SELECT request_id
|
|
656
|
+
FROM benchmark_native_reference_http_requests
|
|
657
|
+
WHERE request_id = ?
|
|
658
|
+
""", (request_id,)).fetchone()
|
|
659
|
+
if mutation is not None:
|
|
660
|
+
raise NativeReferenceQueueBlocked(
|
|
661
|
+
"native_reference.queue.http_request_conflict",
|
|
662
|
+
)
|
|
663
|
+
replay = self.connection.execute("""
|
|
664
|
+
SELECT * FROM benchmark_native_reference_claim_requests
|
|
665
|
+
WHERE request_id = ?
|
|
666
|
+
""", (request_id,)).fetchone()
|
|
667
|
+
if replay is not None:
|
|
668
|
+
try:
|
|
669
|
+
if (
|
|
670
|
+
tuple(replay.keys()) != CLAIM_REQUEST_COLUMNS
|
|
671
|
+
or replay["campaign_id"] != campaign_id
|
|
672
|
+
or replay["editor_id"] != worker_id
|
|
673
|
+
or (
|
|
674
|
+
target_locale is not None
|
|
675
|
+
and replay["target_locale"] != target_locale
|
|
676
|
+
)
|
|
677
|
+
):
|
|
678
|
+
raise ValueError
|
|
679
|
+
row = self.connection.execute("""
|
|
680
|
+
SELECT * FROM benchmark_native_reference_queue
|
|
681
|
+
WHERE work_id = ?
|
|
682
|
+
""", (replay["work_id"],)).fetchone()
|
|
683
|
+
if row is None:
|
|
684
|
+
raise ValueError
|
|
685
|
+
self._validate_row(row, now=now)
|
|
686
|
+
if (
|
|
687
|
+
row["status"] != "leased"
|
|
688
|
+
or row["campaign_id"] != campaign_id
|
|
689
|
+
or row["target_locale"] != replay["target_locale"]
|
|
690
|
+
or row["attempts"] != replay["attempt"]
|
|
691
|
+
or row["lease_owner"] != worker_id
|
|
692
|
+
or row["lease_token"] != replay["lease_token"]
|
|
693
|
+
or row["lease_expires_at"] != replay["lease_expires_at"]
|
|
694
|
+
or row["lease_expires_at"] <= now
|
|
695
|
+
):
|
|
696
|
+
raise NativeReferenceQueueBlocked(
|
|
697
|
+
"native_reference.queue.claim_replay_stale",
|
|
698
|
+
)
|
|
699
|
+
job_payload = _CAMPAIGN._job_payload(
|
|
700
|
+
policy, row["target_locale"], row["suite_case_key"],
|
|
701
|
+
)
|
|
702
|
+
return ClaimedNativeReference(
|
|
703
|
+
work_id=row["work_id"], campaign_id=campaign_id,
|
|
704
|
+
target_locale=row["target_locale"],
|
|
705
|
+
suite_case_key=row["suite_case_key"],
|
|
706
|
+
job_payload=job_payload,
|
|
707
|
+
attempt=row["attempts"],
|
|
708
|
+
max_attempts=row["max_attempts"],
|
|
709
|
+
lease_owner=worker_id,
|
|
710
|
+
lease_token=row["lease_token"],
|
|
711
|
+
lease_expires_at=row["lease_expires_at"],
|
|
712
|
+
)
|
|
713
|
+
except NativeReferenceQueueBlocked:
|
|
714
|
+
raise
|
|
715
|
+
except Exception:
|
|
716
|
+
raise NativeReferenceQueueBlocked(
|
|
717
|
+
"native_reference.queue.claim_request_invalid",
|
|
718
|
+
) from None
|
|
719
|
+
expired = self.connection.execute("""
|
|
720
|
+
SELECT work_id, attempts, max_attempts
|
|
721
|
+
FROM benchmark_native_reference_queue
|
|
722
|
+
WHERE campaign_id = ? AND status = 'leased'
|
|
723
|
+
AND lease_expires_at <= ?
|
|
724
|
+
""", (campaign_id, now)).fetchall()
|
|
725
|
+
for row in expired:
|
|
726
|
+
terminal = row["attempts"] >= row["max_attempts"]
|
|
727
|
+
self.connection.execute("""
|
|
728
|
+
UPDATE benchmark_native_reference_http_requests
|
|
729
|
+
SET status = 'abandoned', updated_at = ?
|
|
730
|
+
WHERE work_id = ? AND attempt = ?
|
|
731
|
+
AND status = 'processing'
|
|
732
|
+
""", (now, row["work_id"], row["attempts"]))
|
|
733
|
+
self.connection.execute("""
|
|
734
|
+
UPDATE benchmark_native_reference_queue
|
|
735
|
+
SET status = ?, next_attempt_at = ?, lease_owner = NULL,
|
|
736
|
+
lease_token = NULL, lease_expires_at = NULL,
|
|
737
|
+
last_error_code = 'native_reference.queue.lease_expired',
|
|
738
|
+
updated_at = ? WHERE work_id = ?
|
|
739
|
+
""", (
|
|
740
|
+
"failed" if terminal else "retry_wait", now, now,
|
|
741
|
+
row["work_id"],
|
|
742
|
+
))
|
|
743
|
+
locale_filter = "" if target_locale is None else "AND target_locale = ?"
|
|
744
|
+
parameters = (
|
|
745
|
+
(campaign_id, now)
|
|
746
|
+
if target_locale is None else (campaign_id, now, target_locale)
|
|
747
|
+
)
|
|
748
|
+
row = self.connection.execute(f"""
|
|
749
|
+
SELECT * FROM benchmark_native_reference_queue
|
|
750
|
+
WHERE campaign_id = ?
|
|
751
|
+
AND status IN ('pending', 'retry_wait')
|
|
752
|
+
AND next_attempt_at <= ? AND attempts < max_attempts
|
|
753
|
+
{locale_filter}
|
|
754
|
+
ORDER BY target_locale, suite_case_key LIMIT 1
|
|
755
|
+
""", parameters).fetchone()
|
|
756
|
+
if row is None:
|
|
757
|
+
return None
|
|
758
|
+
job_payload = _CAMPAIGN._job_payload(
|
|
759
|
+
policy, row["target_locale"], row["suite_case_key"],
|
|
760
|
+
)
|
|
761
|
+
attempt = row["attempts"] + 1
|
|
762
|
+
token = secrets.token_urlsafe(32)
|
|
763
|
+
expires = now + lease_seconds
|
|
764
|
+
changed = self.connection.execute("""
|
|
765
|
+
UPDATE benchmark_native_reference_queue
|
|
766
|
+
SET status = 'leased', attempts = ?, lease_owner = ?,
|
|
767
|
+
lease_token = ?, lease_expires_at = ?,
|
|
768
|
+
last_error_code = NULL, last_error_detail_hash = NULL,
|
|
769
|
+
updated_at = ?
|
|
770
|
+
WHERE work_id = ? AND status IN ('pending', 'retry_wait')
|
|
771
|
+
""", (
|
|
772
|
+
attempt, worker_id, token, expires, now, row["work_id"],
|
|
773
|
+
)).rowcount
|
|
774
|
+
if changed != 1:
|
|
775
|
+
raise NativeReferenceQueueBlocked(
|
|
776
|
+
"native_reference.queue.claim_lost",
|
|
777
|
+
)
|
|
778
|
+
if request_id is not None:
|
|
779
|
+
self.connection.execute("""
|
|
780
|
+
INSERT INTO benchmark_native_reference_claim_requests (
|
|
781
|
+
request_id, campaign_id, editor_id, target_locale,
|
|
782
|
+
work_id, attempt, lease_token, lease_expires_at, created_at
|
|
783
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
784
|
+
""", (
|
|
785
|
+
request_id, campaign_id, worker_id, row["target_locale"],
|
|
786
|
+
row["work_id"], attempt, token, expires, now,
|
|
787
|
+
))
|
|
788
|
+
return ClaimedNativeReference(
|
|
789
|
+
work_id=row["work_id"], campaign_id=campaign_id,
|
|
790
|
+
target_locale=row["target_locale"],
|
|
791
|
+
suite_case_key=row["suite_case_key"],
|
|
792
|
+
job_payload=job_payload,
|
|
793
|
+
attempt=attempt, max_attempts=row["max_attempts"],
|
|
794
|
+
lease_owner=worker_id, lease_token=token,
|
|
795
|
+
lease_expires_at=expires,
|
|
796
|
+
)
|
|
797
|
+
|
|
798
|
+
def _assert_live_locked(
|
|
799
|
+
self, claim: Any, now: float,
|
|
800
|
+
) -> sqlite3.Row:
|
|
801
|
+
if not isinstance(claim, ClaimedNativeReference):
|
|
802
|
+
raise NativeReferenceQueueBlocked(
|
|
803
|
+
"native_reference.queue.claim_invalid",
|
|
804
|
+
)
|
|
805
|
+
row = self.connection.execute("""
|
|
806
|
+
SELECT * FROM benchmark_native_reference_queue WHERE work_id = ?
|
|
807
|
+
""", (claim.work_id,)).fetchone()
|
|
808
|
+
if row is None or (
|
|
809
|
+
row["status"] != "leased"
|
|
810
|
+
or row["campaign_id"] != claim.campaign_id
|
|
811
|
+
or row["target_locale"] != claim.target_locale
|
|
812
|
+
or row["suite_case_key"] != claim.suite_case_key
|
|
813
|
+
or row["attempts"] != claim.attempt
|
|
814
|
+
or row["max_attempts"] != claim.max_attempts
|
|
815
|
+
or row["lease_owner"] != claim.lease_owner
|
|
816
|
+
or row["lease_token"] != claim.lease_token
|
|
817
|
+
or row["lease_expires_at"] != claim.lease_expires_at
|
|
818
|
+
or row["lease_expires_at"] <= now
|
|
819
|
+
):
|
|
820
|
+
raise NativeReferenceQueueBlocked(
|
|
821
|
+
"native_reference.queue.lease_lost",
|
|
822
|
+
)
|
|
823
|
+
return row
|
|
824
|
+
|
|
825
|
+
def assert_live(self, claim: Any, *, now: Any) -> None:
|
|
826
|
+
now = _timestamp(now)
|
|
827
|
+
with _transaction(self.connection):
|
|
828
|
+
self._assert_live_locked(claim, now)
|
|
829
|
+
|
|
830
|
+
@staticmethod
|
|
831
|
+
def _http_request_values(
|
|
832
|
+
claim: Any, operation: Any, request_id: Any, request_sha256: Any,
|
|
833
|
+
) -> tuple[str, str, str, str]:
|
|
834
|
+
if not isinstance(claim, ClaimedNativeReference):
|
|
835
|
+
raise NativeReferenceQueueBlocked(
|
|
836
|
+
"native_reference.queue.claim_invalid",
|
|
837
|
+
)
|
|
838
|
+
operation = _identifier(operation)
|
|
839
|
+
request_id = _identifier(request_id)
|
|
840
|
+
if operation not in HTTP_OPERATIONS:
|
|
841
|
+
raise NativeReferenceQueueBlocked(
|
|
842
|
+
"native_reference.queue.http_request_invalid",
|
|
843
|
+
)
|
|
844
|
+
if (
|
|
845
|
+
not isinstance(request_sha256, str)
|
|
846
|
+
or SHA256.fullmatch(request_sha256) is None
|
|
847
|
+
):
|
|
848
|
+
raise NativeReferenceQueueBlocked(
|
|
849
|
+
"native_reference.queue.http_request_invalid",
|
|
850
|
+
)
|
|
851
|
+
return operation, request_id, request_sha256, claim.target_locale
|
|
852
|
+
|
|
853
|
+
@classmethod
|
|
854
|
+
def _http_result_from_row(cls, row: sqlite3.Row) -> dict[str, Any]:
|
|
855
|
+
try:
|
|
856
|
+
result = json.loads(row["result_json"])
|
|
857
|
+
cls._validate_http_result(row["operation"], result)
|
|
858
|
+
if cls._canonical_json(result) != row["result_json"]:
|
|
859
|
+
raise ValueError
|
|
860
|
+
return result
|
|
861
|
+
except NativeReferenceQueueBlocked:
|
|
862
|
+
raise
|
|
863
|
+
except Exception:
|
|
864
|
+
raise NativeReferenceQueueBlocked(
|
|
865
|
+
"native_reference.queue.state_invalid",
|
|
866
|
+
) from None
|
|
867
|
+
|
|
868
|
+
@staticmethod
|
|
869
|
+
def _assert_http_request_binding(
|
|
870
|
+
row: sqlite3.Row,
|
|
871
|
+
*,
|
|
872
|
+
campaign_id: str,
|
|
873
|
+
editor_id: str,
|
|
874
|
+
target_locale: str,
|
|
875
|
+
operation: str,
|
|
876
|
+
request_id: str,
|
|
877
|
+
request_sha256: str,
|
|
878
|
+
) -> None:
|
|
879
|
+
if (
|
|
880
|
+
tuple(row.keys()) != HTTP_REQUEST_COLUMNS
|
|
881
|
+
or row["request_id"] != request_id
|
|
882
|
+
or row["campaign_id"] != campaign_id
|
|
883
|
+
or row["editor_id"] != editor_id
|
|
884
|
+
or row["target_locale"] != target_locale
|
|
885
|
+
or row["operation"] != operation
|
|
886
|
+
or row["request_sha256"] != request_sha256
|
|
887
|
+
):
|
|
888
|
+
raise NativeReferenceQueueBlocked(
|
|
889
|
+
"native_reference.queue.http_request_conflict",
|
|
890
|
+
)
|
|
891
|
+
|
|
892
|
+
def replay_http_request(
|
|
893
|
+
self,
|
|
894
|
+
policy: Any,
|
|
895
|
+
campaign_id: Any,
|
|
896
|
+
editor_id: Any,
|
|
897
|
+
target_locale: Any,
|
|
898
|
+
operation: Any,
|
|
899
|
+
request_id: Any,
|
|
900
|
+
request_sha256: Any,
|
|
901
|
+
*,
|
|
902
|
+
now: Any,
|
|
903
|
+
) -> dict[str, Any] | None:
|
|
904
|
+
campaign_id = _identifier(campaign_id)
|
|
905
|
+
editor_id = _identifier(editor_id)
|
|
906
|
+
target_locale = _identifier(target_locale)
|
|
907
|
+
operation = _identifier(operation)
|
|
908
|
+
request_id = _identifier(request_id)
|
|
909
|
+
now = _timestamp(now)
|
|
910
|
+
if (
|
|
911
|
+
operation not in HTTP_OPERATIONS
|
|
912
|
+
or not isinstance(request_sha256, str)
|
|
913
|
+
or SHA256.fullmatch(request_sha256) is None
|
|
914
|
+
):
|
|
915
|
+
raise NativeReferenceQueueBlocked(
|
|
916
|
+
"native_reference.queue.http_request_invalid",
|
|
917
|
+
)
|
|
918
|
+
try:
|
|
919
|
+
policy = _CAMPAIGN._BENCHMARK._validate_policy(policy)
|
|
920
|
+
_CAMPAIGN._assert_policy_current(policy, now)
|
|
921
|
+
except Exception:
|
|
922
|
+
raise NativeReferenceQueueBlocked(
|
|
923
|
+
"native_reference.queue.policy_invalid",
|
|
924
|
+
) from None
|
|
925
|
+
with _transaction(self.connection):
|
|
926
|
+
self._verify_binding_locked(policy, campaign_id)
|
|
927
|
+
row = self.connection.execute("""
|
|
928
|
+
SELECT * FROM benchmark_native_reference_http_requests
|
|
929
|
+
WHERE request_id = ?
|
|
930
|
+
""", (request_id,)).fetchone()
|
|
931
|
+
if row is None:
|
|
932
|
+
return None
|
|
933
|
+
self._assert_http_request_binding(
|
|
934
|
+
row,
|
|
935
|
+
campaign_id=campaign_id,
|
|
936
|
+
editor_id=editor_id,
|
|
937
|
+
target_locale=target_locale,
|
|
938
|
+
operation=operation,
|
|
939
|
+
request_id=request_id,
|
|
940
|
+
request_sha256=request_sha256,
|
|
941
|
+
)
|
|
942
|
+
if row["status"] == "abandoned":
|
|
943
|
+
raise NativeReferenceQueueBlocked(
|
|
944
|
+
"native_reference.queue.http_request_replay_stale",
|
|
945
|
+
)
|
|
946
|
+
if row["status"] != "completed":
|
|
947
|
+
raise NativeReferenceQueueBlocked(
|
|
948
|
+
"native_reference.queue.http_request_in_progress",
|
|
949
|
+
)
|
|
950
|
+
return self._http_result_from_row(row)
|
|
951
|
+
|
|
952
|
+
def _begin_http_request_locked(
|
|
953
|
+
self,
|
|
954
|
+
claim: ClaimedNativeReference,
|
|
955
|
+
operation: str,
|
|
956
|
+
request_id: str,
|
|
957
|
+
request_sha256: str,
|
|
958
|
+
now: float,
|
|
959
|
+
) -> dict[str, Any] | None:
|
|
960
|
+
row = self.connection.execute("""
|
|
961
|
+
SELECT * FROM benchmark_native_reference_http_requests
|
|
962
|
+
WHERE request_id = ?
|
|
963
|
+
""", (request_id,)).fetchone()
|
|
964
|
+
if row is not None:
|
|
965
|
+
self._assert_http_request_binding(
|
|
966
|
+
row,
|
|
967
|
+
campaign_id=claim.campaign_id,
|
|
968
|
+
editor_id=claim.lease_owner,
|
|
969
|
+
target_locale=claim.target_locale,
|
|
970
|
+
operation=operation,
|
|
971
|
+
request_id=request_id,
|
|
972
|
+
request_sha256=request_sha256,
|
|
973
|
+
)
|
|
974
|
+
if row["status"] == "abandoned":
|
|
975
|
+
raise NativeReferenceQueueBlocked(
|
|
976
|
+
"native_reference.queue.http_request_replay_stale",
|
|
977
|
+
)
|
|
978
|
+
if row["status"] != "completed":
|
|
979
|
+
raise NativeReferenceQueueBlocked(
|
|
980
|
+
"native_reference.queue.http_request_in_progress",
|
|
981
|
+
)
|
|
982
|
+
return self._http_result_from_row(row)
|
|
983
|
+
claim_request = self.connection.execute("""
|
|
984
|
+
SELECT request_id FROM benchmark_native_reference_claim_requests
|
|
985
|
+
WHERE request_id = ?
|
|
986
|
+
""", (request_id,)).fetchone()
|
|
987
|
+
if claim_request is not None:
|
|
988
|
+
raise NativeReferenceQueueBlocked(
|
|
989
|
+
"native_reference.queue.http_request_conflict",
|
|
990
|
+
)
|
|
991
|
+
if operation == "submit":
|
|
992
|
+
prior = self.connection.execute("""
|
|
993
|
+
SELECT request_id
|
|
994
|
+
FROM benchmark_native_reference_http_requests
|
|
995
|
+
WHERE campaign_id = ? AND work_id = ? AND attempt = ?
|
|
996
|
+
AND operation = 'submit'
|
|
997
|
+
""", (claim.campaign_id, claim.work_id, claim.attempt)).fetchone()
|
|
998
|
+
if prior is not None:
|
|
999
|
+
raise NativeReferenceQueueBlocked(
|
|
1000
|
+
"native_reference.queue.http_request_conflict",
|
|
1001
|
+
)
|
|
1002
|
+
self.connection.execute("""
|
|
1003
|
+
INSERT INTO benchmark_native_reference_http_requests (
|
|
1004
|
+
request_id, campaign_id, editor_id, target_locale, operation,
|
|
1005
|
+
request_sha256, work_id, attempt, lease_token,
|
|
1006
|
+
lease_expires_at, status, created_at, updated_at
|
|
1007
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'processing', ?, ?)
|
|
1008
|
+
""", (
|
|
1009
|
+
request_id, claim.campaign_id, claim.lease_owner,
|
|
1010
|
+
claim.target_locale, operation, request_sha256, claim.work_id,
|
|
1011
|
+
claim.attempt, claim.lease_token, claim.lease_expires_at, now, now,
|
|
1012
|
+
))
|
|
1013
|
+
return None
|
|
1014
|
+
|
|
1015
|
+
def begin_http_submission_request(
|
|
1016
|
+
self,
|
|
1017
|
+
policy: Any,
|
|
1018
|
+
claim: Any,
|
|
1019
|
+
request_id: Any,
|
|
1020
|
+
request_sha256: Any,
|
|
1021
|
+
*,
|
|
1022
|
+
now: Any,
|
|
1023
|
+
) -> dict[str, Any] | None:
|
|
1024
|
+
operation, request_id, request_sha256, _ = self._http_request_values(
|
|
1025
|
+
claim, "submit", request_id, request_sha256,
|
|
1026
|
+
)
|
|
1027
|
+
now = _timestamp(now)
|
|
1028
|
+
try:
|
|
1029
|
+
policy = _CAMPAIGN._BENCHMARK._validate_policy(policy)
|
|
1030
|
+
_CAMPAIGN._assert_policy_current(policy, now)
|
|
1031
|
+
except Exception:
|
|
1032
|
+
raise NativeReferenceQueueBlocked(
|
|
1033
|
+
"native_reference.queue.policy_invalid",
|
|
1034
|
+
) from None
|
|
1035
|
+
with _transaction(self.connection):
|
|
1036
|
+
self._verify_binding_locked(policy, claim.campaign_id)
|
|
1037
|
+
self._assert_live_locked(claim, now)
|
|
1038
|
+
return self._begin_http_request_locked(
|
|
1039
|
+
claim, operation, request_id, request_sha256, now,
|
|
1040
|
+
)
|
|
1041
|
+
|
|
1042
|
+
def _finish_http_request_locked(
|
|
1043
|
+
self,
|
|
1044
|
+
claim: ClaimedNativeReference,
|
|
1045
|
+
operation: str,
|
|
1046
|
+
request_id: str,
|
|
1047
|
+
request_sha256: str,
|
|
1048
|
+
result: dict[str, Any],
|
|
1049
|
+
now: float,
|
|
1050
|
+
) -> None:
|
|
1051
|
+
row = self.connection.execute("""
|
|
1052
|
+
SELECT * FROM benchmark_native_reference_http_requests
|
|
1053
|
+
WHERE request_id = ?
|
|
1054
|
+
""", (request_id,)).fetchone()
|
|
1055
|
+
if row is None:
|
|
1056
|
+
raise NativeReferenceQueueBlocked(
|
|
1057
|
+
"native_reference.queue.http_request_invalid",
|
|
1058
|
+
)
|
|
1059
|
+
self._assert_http_request_binding(
|
|
1060
|
+
row,
|
|
1061
|
+
campaign_id=claim.campaign_id,
|
|
1062
|
+
editor_id=claim.lease_owner,
|
|
1063
|
+
target_locale=claim.target_locale,
|
|
1064
|
+
operation=operation,
|
|
1065
|
+
request_id=request_id,
|
|
1066
|
+
request_sha256=request_sha256,
|
|
1067
|
+
)
|
|
1068
|
+
if row["status"] != "processing":
|
|
1069
|
+
raise NativeReferenceQueueBlocked(
|
|
1070
|
+
"native_reference.queue.http_request_conflict",
|
|
1071
|
+
)
|
|
1072
|
+
result = self._validate_http_result(operation, result)
|
|
1073
|
+
encoded = self._canonical_json(result)
|
|
1074
|
+
self.connection.execute("""
|
|
1075
|
+
UPDATE benchmark_native_reference_http_requests
|
|
1076
|
+
SET status = 'completed', result_json = ?, result_sha256 = ?,
|
|
1077
|
+
updated_at = ?
|
|
1078
|
+
WHERE request_id = ? AND status = 'processing'
|
|
1079
|
+
""", (
|
|
1080
|
+
encoded,
|
|
1081
|
+
hashlib.sha256(encoded.encode("utf-8")).hexdigest(),
|
|
1082
|
+
now,
|
|
1083
|
+
request_id,
|
|
1084
|
+
))
|
|
1085
|
+
|
|
1086
|
+
def renew(
|
|
1087
|
+
self, claim: Any, *, now: Any, lease_seconds: Any, policy: Any = None,
|
|
1088
|
+
request_id: Any = None, request_sha256: Any = None,
|
|
1089
|
+
) -> ClaimedNativeReference:
|
|
1090
|
+
now = _timestamp(now)
|
|
1091
|
+
expires = now + _duration(lease_seconds)
|
|
1092
|
+
http_values = None
|
|
1093
|
+
if request_id is not None or request_sha256 is not None:
|
|
1094
|
+
if policy is None:
|
|
1095
|
+
raise NativeReferenceQueueBlocked(
|
|
1096
|
+
"native_reference.queue.http_request_invalid",
|
|
1097
|
+
)
|
|
1098
|
+
http_values = self._http_request_values(
|
|
1099
|
+
claim, "renew", request_id, request_sha256,
|
|
1100
|
+
)
|
|
1101
|
+
try:
|
|
1102
|
+
policy = _CAMPAIGN._BENCHMARK._validate_policy(policy)
|
|
1103
|
+
_CAMPAIGN._assert_policy_current(policy, now)
|
|
1104
|
+
except Exception:
|
|
1105
|
+
raise NativeReferenceQueueBlocked(
|
|
1106
|
+
"native_reference.queue.policy_invalid",
|
|
1107
|
+
) from None
|
|
1108
|
+
with _transaction(self.connection):
|
|
1109
|
+
if http_values is not None:
|
|
1110
|
+
self._verify_binding_locked(policy, claim.campaign_id)
|
|
1111
|
+
replay = self._begin_http_request_locked(
|
|
1112
|
+
claim, *http_values[:3], now,
|
|
1113
|
+
)
|
|
1114
|
+
if replay is not None:
|
|
1115
|
+
return ClaimedNativeReference(**{
|
|
1116
|
+
**asdict(claim),
|
|
1117
|
+
"lease_expires_at": replay["lease_expires_at"],
|
|
1118
|
+
})
|
|
1119
|
+
self._assert_live_locked(claim, now)
|
|
1120
|
+
self.connection.execute("""
|
|
1121
|
+
UPDATE benchmark_native_reference_queue
|
|
1122
|
+
SET lease_expires_at = ?, updated_at = ? WHERE work_id = ?
|
|
1123
|
+
""", (expires, now, claim.work_id))
|
|
1124
|
+
self.connection.execute("""
|
|
1125
|
+
UPDATE benchmark_native_reference_claim_requests
|
|
1126
|
+
SET lease_expires_at = ?
|
|
1127
|
+
WHERE work_id = ? AND attempt = ? AND editor_id = ?
|
|
1128
|
+
AND lease_token = ?
|
|
1129
|
+
""", (
|
|
1130
|
+
expires, claim.work_id, claim.attempt,
|
|
1131
|
+
claim.lease_owner, claim.lease_token,
|
|
1132
|
+
))
|
|
1133
|
+
if http_values is not None:
|
|
1134
|
+
self._finish_http_request_locked(
|
|
1135
|
+
claim,
|
|
1136
|
+
*http_values[:3],
|
|
1137
|
+
{"lease_expires_at": expires},
|
|
1138
|
+
now,
|
|
1139
|
+
)
|
|
1140
|
+
return ClaimedNativeReference(**{
|
|
1141
|
+
**asdict(claim), "lease_expires_at": expires,
|
|
1142
|
+
})
|
|
1143
|
+
|
|
1144
|
+
def complete(
|
|
1145
|
+
self, policy: Any, claim: Any, artifact_sha256: Any, *, now: Any,
|
|
1146
|
+
request_id: Any = None, request_sha256: Any = None,
|
|
1147
|
+
) -> NativeReferenceQueueOutcome:
|
|
1148
|
+
if not isinstance(claim, ClaimedNativeReference):
|
|
1149
|
+
raise NativeReferenceQueueBlocked(
|
|
1150
|
+
"native_reference.queue.claim_invalid",
|
|
1151
|
+
)
|
|
1152
|
+
if not isinstance(artifact_sha256, str) or SHA256.fullmatch(
|
|
1153
|
+
artifact_sha256,
|
|
1154
|
+
) is None:
|
|
1155
|
+
raise NativeReferenceQueueBlocked(
|
|
1156
|
+
"native_reference.queue.artifact_invalid",
|
|
1157
|
+
)
|
|
1158
|
+
now = _timestamp(now)
|
|
1159
|
+
http_values = None
|
|
1160
|
+
if request_id is not None or request_sha256 is not None:
|
|
1161
|
+
http_values = self._http_request_values(
|
|
1162
|
+
claim, "submit", request_id, request_sha256,
|
|
1163
|
+
)
|
|
1164
|
+
try:
|
|
1165
|
+
policy = _CAMPAIGN._BENCHMARK._validate_policy(policy)
|
|
1166
|
+
_CAMPAIGN._assert_policy_current(policy, now)
|
|
1167
|
+
except Exception:
|
|
1168
|
+
raise NativeReferenceQueueBlocked(
|
|
1169
|
+
"native_reference.queue.policy_invalid",
|
|
1170
|
+
) from None
|
|
1171
|
+
with _transaction(self.connection):
|
|
1172
|
+
self._verify_binding_locked(policy, claim.campaign_id)
|
|
1173
|
+
self._assert_live_locked(claim, now)
|
|
1174
|
+
self.connection.execute("""
|
|
1175
|
+
UPDATE benchmark_native_reference_queue
|
|
1176
|
+
SET status = 'succeeded', lease_owner = NULL,
|
|
1177
|
+
lease_token = NULL, lease_expires_at = NULL,
|
|
1178
|
+
last_error_code = NULL, last_error_detail_hash = NULL,
|
|
1179
|
+
artifact_sha256 = ?, updated_at = ? WHERE work_id = ?
|
|
1180
|
+
""", (artifact_sha256, now, claim.work_id))
|
|
1181
|
+
outcome = self._outcome(claim.work_id)
|
|
1182
|
+
if http_values is not None:
|
|
1183
|
+
self._finish_http_request_locked(
|
|
1184
|
+
claim,
|
|
1185
|
+
*http_values[:3],
|
|
1186
|
+
self._outcome_payload(outcome),
|
|
1187
|
+
now,
|
|
1188
|
+
)
|
|
1189
|
+
return outcome
|
|
1190
|
+
|
|
1191
|
+
def transition_failure(
|
|
1192
|
+
self, policy: Any, claim: Any, code: Any, *, retryable: bool,
|
|
1193
|
+
delay_seconds: Any = 0, detail: str | None = None, now: Any,
|
|
1194
|
+
request_id: Any = None, request_sha256: Any = None,
|
|
1195
|
+
) -> NativeReferenceQueueOutcome:
|
|
1196
|
+
if not isinstance(claim, ClaimedNativeReference):
|
|
1197
|
+
raise NativeReferenceQueueBlocked(
|
|
1198
|
+
"native_reference.queue.claim_invalid",
|
|
1199
|
+
)
|
|
1200
|
+
code = _identifier(code)
|
|
1201
|
+
if ERROR_CODE.fullmatch(code) is None or not isinstance(retryable, bool):
|
|
1202
|
+
raise NativeReferenceQueueBlocked(
|
|
1203
|
+
"native_reference.queue.failure_invalid",
|
|
1204
|
+
)
|
|
1205
|
+
delay = _duration(delay_seconds, allow_zero=True)
|
|
1206
|
+
now = _timestamp(now)
|
|
1207
|
+
http_values = None
|
|
1208
|
+
if request_id is not None or request_sha256 is not None:
|
|
1209
|
+
http_values = self._http_request_values(
|
|
1210
|
+
claim, "submit", request_id, request_sha256,
|
|
1211
|
+
)
|
|
1212
|
+
if detail is not None and (
|
|
1213
|
+
not isinstance(detail, str) or "\x00" in detail
|
|
1214
|
+
or not unicodedata.is_normalized("NFC", detail)
|
|
1215
|
+
):
|
|
1216
|
+
raise NativeReferenceQueueBlocked(
|
|
1217
|
+
"native_reference.queue.failure_invalid",
|
|
1218
|
+
)
|
|
1219
|
+
detail_hash = (
|
|
1220
|
+
hashlib.sha256(detail.encode("utf-8")).hexdigest()
|
|
1221
|
+
if detail is not None else None
|
|
1222
|
+
)
|
|
1223
|
+
try:
|
|
1224
|
+
policy = _CAMPAIGN._BENCHMARK._validate_policy(policy)
|
|
1225
|
+
except Exception:
|
|
1226
|
+
raise NativeReferenceQueueBlocked(
|
|
1227
|
+
"native_reference.queue.policy_invalid",
|
|
1228
|
+
) from None
|
|
1229
|
+
with _transaction(self.connection):
|
|
1230
|
+
self._verify_binding_locked(policy, claim.campaign_id)
|
|
1231
|
+
row = self._assert_live_locked(claim, now)
|
|
1232
|
+
terminal = not retryable or row["attempts"] >= row["max_attempts"]
|
|
1233
|
+
self.connection.execute("""
|
|
1234
|
+
UPDATE benchmark_native_reference_queue
|
|
1235
|
+
SET status = ?, next_attempt_at = ?, lease_owner = NULL,
|
|
1236
|
+
lease_token = NULL, lease_expires_at = NULL,
|
|
1237
|
+
last_error_code = ?, last_error_detail_hash = ?,
|
|
1238
|
+
updated_at = ? WHERE work_id = ?
|
|
1239
|
+
""", (
|
|
1240
|
+
"failed" if terminal else "retry_wait",
|
|
1241
|
+
now if terminal else now + delay, code, detail_hash, now,
|
|
1242
|
+
claim.work_id,
|
|
1243
|
+
))
|
|
1244
|
+
outcome = self._outcome(claim.work_id)
|
|
1245
|
+
if http_values is not None:
|
|
1246
|
+
self._finish_http_request_locked(
|
|
1247
|
+
claim,
|
|
1248
|
+
*http_values[:3],
|
|
1249
|
+
self._outcome_payload(outcome),
|
|
1250
|
+
now,
|
|
1251
|
+
)
|
|
1252
|
+
return outcome
|
|
1253
|
+
|
|
1254
|
+
@staticmethod
|
|
1255
|
+
def _outcome_payload(
|
|
1256
|
+
outcome: NativeReferenceQueueOutcome,
|
|
1257
|
+
) -> dict[str, Any]:
|
|
1258
|
+
return {field: getattr(outcome, field) for field in OUTCOME_FIELDS}
|
|
1259
|
+
|
|
1260
|
+
@staticmethod
|
|
1261
|
+
def outcome_from_payload(payload: Any) -> NativeReferenceQueueOutcome:
|
|
1262
|
+
payload = NativeReferenceWorkQueue._validate_http_result(
|
|
1263
|
+
"submit", payload,
|
|
1264
|
+
)
|
|
1265
|
+
return NativeReferenceQueueOutcome(**payload)
|
|
1266
|
+
|
|
1267
|
+
def _outcome(self, work_id: str) -> NativeReferenceQueueOutcome:
|
|
1268
|
+
row = self.connection.execute("""
|
|
1269
|
+
SELECT * FROM benchmark_native_reference_queue WHERE work_id = ?
|
|
1270
|
+
""", (work_id,)).fetchone()
|
|
1271
|
+
if row is None:
|
|
1272
|
+
raise NativeReferenceQueueBlocked(
|
|
1273
|
+
"native_reference.queue.state_invalid",
|
|
1274
|
+
)
|
|
1275
|
+
return NativeReferenceQueueOutcome(
|
|
1276
|
+
work_id=row["work_id"], target_locale=row["target_locale"],
|
|
1277
|
+
suite_case_key=row["suite_case_key"], status=row["status"],
|
|
1278
|
+
attempt=row["attempts"], max_attempts=row["max_attempts"],
|
|
1279
|
+
next_attempt_at=row["next_attempt_at"],
|
|
1280
|
+
error_code=row["last_error_code"],
|
|
1281
|
+
error_detail_hash=row["last_error_detail_hash"],
|
|
1282
|
+
artifact_sha256=row["artifact_sha256"],
|
|
1283
|
+
)
|
|
1284
|
+
|
|
1285
|
+
def status(self, policy: Any, campaign_id: str) -> dict[str, Any]:
|
|
1286
|
+
try:
|
|
1287
|
+
policy = _CAMPAIGN._BENCHMARK._validate_policy(policy)
|
|
1288
|
+
with _transaction(self.connection):
|
|
1289
|
+
self._verify_binding_locked(policy, campaign_id)
|
|
1290
|
+
rows = self.connection.execute("""
|
|
1291
|
+
SELECT status, COUNT(*) AS count
|
|
1292
|
+
FROM benchmark_native_reference_queue
|
|
1293
|
+
WHERE campaign_id = ? GROUP BY status
|
|
1294
|
+
""", (campaign_id,)).fetchall()
|
|
1295
|
+
errors = self.connection.execute("""
|
|
1296
|
+
SELECT last_error_code, COUNT(*) AS count
|
|
1297
|
+
FROM benchmark_native_reference_queue
|
|
1298
|
+
WHERE campaign_id = ? AND last_error_code IS NOT NULL
|
|
1299
|
+
GROUP BY last_error_code ORDER BY last_error_code
|
|
1300
|
+
""", (campaign_id,)).fetchall()
|
|
1301
|
+
http_requests = self.connection.execute("""
|
|
1302
|
+
SELECT status, COUNT(*) AS count
|
|
1303
|
+
FROM benchmark_native_reference_http_requests
|
|
1304
|
+
WHERE campaign_id = ? GROUP BY status
|
|
1305
|
+
""", (campaign_id,)).fetchall()
|
|
1306
|
+
except NativeReferenceQueueBlocked:
|
|
1307
|
+
raise
|
|
1308
|
+
except Exception:
|
|
1309
|
+
raise NativeReferenceQueueBlocked(
|
|
1310
|
+
"native_reference.queue.state_invalid",
|
|
1311
|
+
) from None
|
|
1312
|
+
counts = {name: 0 for name in STATUSES}
|
|
1313
|
+
counts.update({row["status"]: row["count"] for row in rows})
|
|
1314
|
+
http_counts = {name: 0 for name in HTTP_REQUEST_STATUSES}
|
|
1315
|
+
http_counts.update({row["status"]: row["count"] for row in http_requests})
|
|
1316
|
+
return {
|
|
1317
|
+
"schema": SCHEMA,
|
|
1318
|
+
"campaign_id": campaign_id,
|
|
1319
|
+
"work_count": len(_CAMPAIGN._expected_work(policy)),
|
|
1320
|
+
"counts": counts,
|
|
1321
|
+
"error_counts": {
|
|
1322
|
+
row["last_error_code"]: row["count"] for row in errors
|
|
1323
|
+
},
|
|
1324
|
+
"http_request_counts": http_counts,
|
|
1325
|
+
"complete": counts["succeeded"] == len(
|
|
1326
|
+
_CAMPAIGN._expected_work(policy)
|
|
1327
|
+
),
|
|
1328
|
+
"blocked": counts["failed"] > 0,
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
def health(
|
|
1332
|
+
self, policy: Any, campaign_id: str, *, now: Any,
|
|
1333
|
+
stale_after_seconds: Any = 3600,
|
|
1334
|
+
) -> NativeReferenceQueueHealth:
|
|
1335
|
+
now = _timestamp(now)
|
|
1336
|
+
stale_after_seconds = _timestamp(stale_after_seconds)
|
|
1337
|
+
if not 0 < stale_after_seconds <= MAX_STALE_SECONDS:
|
|
1338
|
+
raise NativeReferenceQueueBlocked(
|
|
1339
|
+
"native_reference.queue.stale_threshold_invalid",
|
|
1340
|
+
)
|
|
1341
|
+
counts = {name: 0 for name in STATUSES}
|
|
1342
|
+
reasons: set[str] = set()
|
|
1343
|
+
last_progress: float | None = None
|
|
1344
|
+
try:
|
|
1345
|
+
policy = _CAMPAIGN._BENCHMARK._validate_policy(policy)
|
|
1346
|
+
if now > policy.valid_until:
|
|
1347
|
+
reasons.add("native_reference.queue.policy_expired")
|
|
1348
|
+
if self.connection.in_transaction:
|
|
1349
|
+
raise ValueError
|
|
1350
|
+
self.connection.execute("BEGIN")
|
|
1351
|
+
try:
|
|
1352
|
+
self._verify_binding_locked(policy, campaign_id)
|
|
1353
|
+
rows = self.connection.execute("""
|
|
1354
|
+
SELECT * FROM benchmark_native_reference_queue
|
|
1355
|
+
WHERE campaign_id = ? ORDER BY target_locale, suite_case_key
|
|
1356
|
+
""", (campaign_id,)).fetchall()
|
|
1357
|
+
if len(rows) != len(_CAMPAIGN._expected_work(policy)):
|
|
1358
|
+
raise ValueError
|
|
1359
|
+
for row in rows:
|
|
1360
|
+
if tuple(row.keys()) != TABLE_COLUMNS or row["status"] not in counts:
|
|
1361
|
+
raise ValueError
|
|
1362
|
+
counts[row["status"]] += 1
|
|
1363
|
+
attempts = row["attempts"]
|
|
1364
|
+
maximum = row["max_attempts"]
|
|
1365
|
+
created = _timestamp(row["created_at"])
|
|
1366
|
+
updated = _timestamp(row["updated_at"])
|
|
1367
|
+
next_at = _timestamp(row["next_attempt_at"])
|
|
1368
|
+
if (
|
|
1369
|
+
isinstance(attempts, bool) or not isinstance(attempts, int)
|
|
1370
|
+
or isinstance(maximum, bool) or not isinstance(maximum, int)
|
|
1371
|
+
or not 0 <= attempts <= maximum <= MAX_ATTEMPTS
|
|
1372
|
+
or created > updated or updated > now
|
|
1373
|
+
):
|
|
1374
|
+
raise ValueError
|
|
1375
|
+
last_progress = max(
|
|
1376
|
+
updated,
|
|
1377
|
+
last_progress if last_progress is not None else updated,
|
|
1378
|
+
)
|
|
1379
|
+
leased = row["status"] == "leased"
|
|
1380
|
+
lease_values = (
|
|
1381
|
+
row["lease_owner"], row["lease_token"],
|
|
1382
|
+
row["lease_expires_at"],
|
|
1383
|
+
)
|
|
1384
|
+
if leased:
|
|
1385
|
+
_identifier(row["lease_owner"])
|
|
1386
|
+
_identifier(row["lease_token"])
|
|
1387
|
+
if attempts < 1:
|
|
1388
|
+
raise ValueError
|
|
1389
|
+
if _timestamp(row["lease_expires_at"]) <= now:
|
|
1390
|
+
reasons.add("native_reference.queue.lease_expired")
|
|
1391
|
+
elif any(value is not None for value in lease_values):
|
|
1392
|
+
raise ValueError
|
|
1393
|
+
code = row["last_error_code"]
|
|
1394
|
+
detail = row["last_error_detail_hash"]
|
|
1395
|
+
artifact = row["artifact_sha256"]
|
|
1396
|
+
if code is not None:
|
|
1397
|
+
if not isinstance(code, str) or ERROR_CODE.fullmatch(code) is None:
|
|
1398
|
+
raise ValueError
|
|
1399
|
+
reasons.add("native_reference.queue.error." + code)
|
|
1400
|
+
if detail is not None and (
|
|
1401
|
+
not isinstance(detail, str) or SHA256.fullmatch(detail) is None
|
|
1402
|
+
):
|
|
1403
|
+
raise ValueError
|
|
1404
|
+
if row["status"] == "succeeded":
|
|
1405
|
+
if code is not None or detail is not None or attempts < 1:
|
|
1406
|
+
raise ValueError
|
|
1407
|
+
if not isinstance(artifact, str) or SHA256.fullmatch(artifact) is None:
|
|
1408
|
+
raise ValueError
|
|
1409
|
+
elif artifact is not None:
|
|
1410
|
+
raise ValueError
|
|
1411
|
+
if row["status"] == "pending" and (
|
|
1412
|
+
attempts != 0 or code is not None or detail is not None
|
|
1413
|
+
):
|
|
1414
|
+
raise ValueError
|
|
1415
|
+
if row["status"] in {"retry_wait", "failed"} and (
|
|
1416
|
+
attempts < 1 or code is None
|
|
1417
|
+
):
|
|
1418
|
+
raise ValueError
|
|
1419
|
+
if row["status"] == "retry_wait" and next_at <= now:
|
|
1420
|
+
reasons.add("native_reference.queue.retry_due")
|
|
1421
|
+
if last_progress is None:
|
|
1422
|
+
raise ValueError
|
|
1423
|
+
if counts["failed"]:
|
|
1424
|
+
reasons.add("native_reference.queue.failed")
|
|
1425
|
+
if (
|
|
1426
|
+
counts["succeeded"] < len(rows)
|
|
1427
|
+
and now - last_progress > stale_after_seconds
|
|
1428
|
+
):
|
|
1429
|
+
reasons.add("native_reference.queue.stalled")
|
|
1430
|
+
finally:
|
|
1431
|
+
self.connection.rollback()
|
|
1432
|
+
status = (
|
|
1433
|
+
"blocked" if counts["failed"] or any(
|
|
1434
|
+
reason in {
|
|
1435
|
+
"native_reference.queue.policy_expired",
|
|
1436
|
+
"native_reference.queue.failed",
|
|
1437
|
+
} for reason in reasons
|
|
1438
|
+
)
|
|
1439
|
+
else "degraded" if reasons else "healthy"
|
|
1440
|
+
)
|
|
1441
|
+
except Exception:
|
|
1442
|
+
reasons = {"native_reference.queue.state_invalid"}
|
|
1443
|
+
status = "blocked"
|
|
1444
|
+
return NativeReferenceQueueHealth(
|
|
1445
|
+
campaign_id=campaign_id, status=status,
|
|
1446
|
+
reasons=tuple(sorted(reasons)),
|
|
1447
|
+
counts=tuple((name, counts[name]) for name in STATUSES),
|
|
1448
|
+
work_count=sum(counts.values()), last_progress_at=last_progress,
|
|
1449
|
+
)
|