oh-my-opencode 4.19.3 → 5.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/command/get-unpublished-changes.md +2 -0
- package/.agents/command/omomomo.md +1 -1
- package/.agents/command/publish.md +7 -0
- package/.agents/skills/get-unpublished-changes/SKILL.md +2 -0
- package/.agents/skills/hyperplan/SKILL.md +3 -3
- package/.agents/skills/omomomo/SKILL.md +1 -1
- package/.agents/skills/publish/SKILL.md +7 -0
- package/.opencode/command/get-unpublished-changes.md +2 -0
- package/.opencode/command/omomomo.md +1 -1
- package/.opencode/command/publish.md +7 -0
- package/.opencode/skills/hyperplan/SKILL.md +3 -3
- package/README.ja.md +1 -1
- package/README.ko.md +1 -1
- package/README.md +6 -5
- package/README.ru.md +1 -1
- package/README.zh-cn.md +1 -1
- package/bin/oh-my-opencode.js +14 -1
- package/bin/oh-my-opencode.test.ts +21 -0
- package/dist/agents/sisyphus-junior/agent.d.ts +1 -1
- package/dist/cli/doctor/checks/deprecated-reasoning-keys.d.ts +2 -0
- package/dist/cli/index.js +2925 -1919
- package/dist/cli-node/index.js +2925 -1919
- package/dist/config/schema/agent-overrides.d.ts +1343 -15
- package/dist/config/schema/categories.d.ts +132 -0
- package/dist/config/schema/fallback-models.d.ts +50 -0
- package/dist/config/schema/oh-my-opencode-config.d.ts +1322 -11
- package/dist/config-migration/index.d.ts +1 -0
- package/dist/config-migration/migration-plans.d.ts +1 -0
- package/dist/config-migration/reasoning-unification.d.ts +3 -0
- package/dist/features/monitor/batcher.d.ts +3 -1
- package/dist/features/monitor/manager-internals.d.ts +1 -0
- package/dist/features/monitor/output-injector-types.d.ts +2 -0
- package/dist/features/monitor/output-injector.d.ts +6 -0
- package/dist/features/team-mode/tools/lifecycle-test-fixture.d.ts +2 -0
- package/dist/hooks/codegraph-bootstrap/command-runner.d.ts +1 -0
- package/dist/hooks/model-fallback/next-fallback.d.ts +1 -0
- package/dist/hooks/runtime-fallback/constants.d.ts +1 -1
- package/dist/hooks/todo-continuation-enforcer/types.d.ts +1 -0
- package/dist/hooks/todo-continuation-enforcer/unrecoverable-request-error.d.ts +9 -0
- package/dist/hooks/tool-pair-validator/hook.test-support.d.ts +29 -0
- package/dist/hooks/tool-pair-validator/tool-part-ids.d.ts +14 -5
- package/dist/hooks/tool-pair-validator/tool-result-repair.d.ts +4 -3
- package/dist/hooks/tool-pair-validator/types.d.ts +5 -22
- package/dist/index.js +4864 -4011
- package/dist/mcp/lsp.d.ts +1 -0
- package/dist/oh-my-opencode.schema.json +3221 -222
- package/dist/plugin-handlers/prometheus-agent-config-builder.d.ts +1 -0
- package/dist/shared/agent-variant.d.ts +11 -0
- package/dist/shared/session-prompt-params-helpers.d.ts +6 -1
- package/dist/shared/tmux/constants.d.ts +1 -1
- package/dist/skills/ast-grep/SOURCE +1 -1
- package/dist/skills/ast-grep/install.ps1 +2 -2
- package/dist/skills/ast-grep/install.sh +1 -1
- package/dist/skills/ast-grep/references/install.md +2 -2
- package/dist/skills/ast-grep/tests/smoke.sh +1 -1
- package/dist/skills/coding-agent-sessions/SKILL.md +4 -3
- package/dist/skills/coding-agent-sessions/references/all-platforms.md +3 -1
- package/dist/skills/coding-agent-sessions/scripts/agent_sessions/aside_scanner.py +140 -0
- package/dist/skills/coding-agent-sessions/scripts/agent_sessions/scanners.py +3 -0
- package/dist/skills/data-scientist/SKILL.md +243 -0
- package/dist/skills/data-scientist/references/common-scenarios.md +176 -0
- package/dist/skills/data-scientist/references/execution-templates.md +197 -0
- package/dist/skills/data-scientist/references/integration-patterns.md +153 -0
- package/dist/skills/data-scientist/references/performance-benchmarks.md +37 -0
- package/dist/skills/data-scientist/references/uv-setup.md +78 -0
- package/dist/skills/data-scientist/scripts/quick-query.py +111 -0
- package/dist/skills/data-scientist/scripts/setup-uv.ps1 +53 -0
- package/dist/skills/data-scientist/scripts/setup-uv.sh +60 -0
- package/dist/skills/debugging/SKILL.md +1 -1
- package/dist/skills/programming/SKILL.md +1 -2
- package/dist/skills/start-work/SKILL.md +54 -9
- package/dist/skills/ultimate-browsing/SKILL.md +2 -2
- package/dist/skills/ultimate-browsing/engine/__main__.py +8 -1
- package/dist/skills/ultimate-browsing/engine/bias_check.py +11 -0
- package/dist/skills/ultimate-browsing/engine/fetch_chain.py +90 -52
- package/dist/skills/ultimate-browsing/engine/result_schema.py +10 -1
- package/dist/skills/ultimate-browsing/engine/surrogate.py +214 -0
- package/dist/skills/ultimate-browsing/engine/surrogates.yaml +60 -0
- package/dist/skills/ultimate-browsing/engine/tests/fixtures/amp_redirect_stub.html +7 -0
- package/dist/skills/ultimate-browsing/engine/tests/fixtures/search_interstitial.html +19 -0
- package/dist/skills/ultimate-browsing/engine/tests/fixtures/wayback_available.json +1 -0
- package/dist/skills/ultimate-browsing/engine/tests/fixtures/wayback_snapshot.html +1128 -0
- package/dist/skills/ultimate-browsing/engine/tests/test_surrogate.py +252 -0
- package/dist/skills/ultimate-browsing/engine/tests/test_surrogate_validators.py +78 -0
- package/dist/skills/ultimate-browsing/engine/validators.py +46 -0
- package/dist/skills/ultimate-browsing/engine/waf_detector.py +1 -1
- package/dist/skills/ultimate-browsing/engine/waf_profiles.yaml +10 -5
- package/dist/skills/ultimate-browsing/references/agent-reach/social.md +1 -1
- package/dist/skills/ultimate-browsing/references/chrome-stealth.md +13 -11
- package/dist/skills/ultimate-browsing/references/insane-search/README.md +4 -4
- package/dist/skills/ultimate-browsing/references/insane-search/cache-archive.md +51 -50
- package/dist/skills/ultimate-browsing/references/insane-search/fallback.md +1 -1
- package/dist/skills/ultimate-browsing/references/insane-search/jina.md +8 -2
- package/dist/skills/ultimate-browsing/references/insane-search/naver.md +1 -1
- package/dist/skills/ultimate-browsing/references/insane-search/twitter.md +3 -3
- package/dist/skills/ulw-plan/SKILL.md +2 -2
- package/dist/skills/ulw-plan/references/full-workflow.md +3 -3
- package/dist/skills/ulw-plan/scripts/scaffold-plan.mjs +1 -1
- package/dist/skills/ulw-research/SKILL.md +128 -12
- package/dist/tools/delegate-task/builtin-categories.d.ts +1 -0
- package/dist/tools/delegate-task/builtin-category-definition.d.ts +1 -0
- package/dist/tools/delegate-task/constants.d.ts +1 -1
- package/dist/tui.js +1271 -1108
- package/docs/reference/web-terminal-visual-qa.md +1 -1
- package/package.json +27 -19
- package/packages/lsp-core/src/lsp/client-diagnostics-freshness.integration.test.ts +2 -2
- package/packages/lsp-core/src/lsp/connection.ts +1 -1
- package/packages/lsp-daemon/dist/cli.js +27 -13
- package/packages/lsp-daemon/dist/client.js +49 -35
- package/packages/lsp-daemon/dist/ensure-daemon.d.ts +1 -0
- package/packages/lsp-daemon/dist/ensure-daemon.js +18 -5
- package/packages/lsp-daemon/dist/index.js +34 -20
- package/packages/lsp-tools-mcp/dist/cli.js +1 -1
- package/packages/lsp-tools-mcp/dist/lsp/manager.js +1 -1
- package/packages/lsp-tools-mcp/dist/mcp.js +1 -1
- package/packages/lsp-tools-mcp/dist/tools.js +1 -1
- package/packages/omo-codex/plugin/.codex-plugin/plugin.json +1 -1
- package/packages/omo-codex/plugin/components/bootstrap/dist/cli.js +289 -95
- package/packages/omo-codex/plugin/components/bootstrap/hooks/hooks.json +1 -1
- package/packages/omo-codex/plugin/components/bootstrap/package.json +1 -1
- package/packages/omo-codex/plugin/components/bootstrap/src/setup.ts +7 -7
- package/packages/omo-codex/plugin/components/bootstrap/src/worker.ts +3 -0
- package/packages/omo-codex/plugin/components/codegraph/dist/cli.js +385 -64
- package/packages/omo-codex/plugin/components/codegraph/dist/serve.js +336 -47
- package/packages/omo-codex/plugin/components/codegraph/package.json +1 -1
- package/packages/omo-codex/plugin/components/codegraph/test/hook.test.ts +6 -0
- package/packages/omo-codex/plugin/components/comment-checker/hooks/hooks.json +1 -1
- package/packages/omo-codex/plugin/components/comment-checker/package.json +1 -1
- package/packages/omo-codex/plugin/components/git-bash/hooks/hooks.json +2 -2
- package/packages/omo-codex/plugin/components/git-bash/package.json +1 -1
- package/packages/omo-codex/plugin/components/lazycodex-executor-verify/hooks/hooks.json +1 -1
- package/packages/omo-codex/plugin/components/lazycodex-executor-verify/package.json +1 -1
- package/packages/omo-codex/plugin/components/lsp/dist/.omo-runtime-manifest.json +3 -3
- package/packages/omo-codex/plugin/components/lsp/dist/cli.js +56 -42
- package/packages/omo-codex/plugin/components/lsp/hooks/hooks.json +2 -2
- package/packages/omo-codex/plugin/components/lsp/package.json +1 -1
- package/packages/omo-codex/plugin/components/rules/dist/cli.js +1 -1
- package/packages/omo-codex/plugin/components/rules/hooks/hooks.json +4 -4
- package/packages/omo-codex/plugin/components/rules/package.json +1 -1
- package/packages/omo-codex/plugin/components/rules/src/post-compact-budget.ts +1 -1
- package/packages/omo-codex/plugin/components/start-work-continuation/AGENTS.md +1 -0
- package/packages/omo-codex/plugin/components/start-work-continuation/README.md +5 -1
- package/packages/omo-codex/plugin/components/start-work-continuation/directive.md +2 -1
- package/packages/omo-codex/plugin/components/start-work-continuation/dist/cli.js +18 -0
- package/packages/omo-codex/plugin/components/start-work-continuation/hooks/hooks.json +2 -2
- package/packages/omo-codex/plugin/components/start-work-continuation/package.json +1 -1
- package/packages/omo-codex/plugin/components/start-work-continuation/src/codex-hook.ts +21 -0
- package/packages/omo-codex/plugin/components/start-work-continuation/test/codex-hook.test.ts +105 -0
- package/packages/omo-codex/plugin/components/teammode/hooks/hooks.json +1 -1
- package/packages/omo-codex/plugin/components/teammode/package.json +1 -1
- package/packages/omo-codex/plugin/components/telemetry/hooks/hooks.json +1 -1
- package/packages/omo-codex/plugin/components/telemetry/package.json +1 -1
- package/packages/omo-codex/plugin/components/ultrawork/CHANGELOG.md +2 -0
- package/packages/omo-codex/plugin/components/ultrawork/agents/lazycodex-code-reviewer.toml +1 -1
- package/packages/omo-codex/plugin/components/ultrawork/agents/lazycodex-gate-reviewer.toml +1 -1
- package/packages/omo-codex/plugin/components/ultrawork/agents/lazycodex-qa-executor.toml +1 -1
- package/packages/omo-codex/plugin/components/ultrawork/agents/lazycodex-worker-high.toml +1 -1
- package/packages/omo-codex/plugin/components/ultrawork/agents/lazycodex-worker-low.toml +1 -1
- package/packages/omo-codex/plugin/components/ultrawork/agents/lazycodex-worker-medium.toml +1 -1
- package/packages/omo-codex/plugin/components/ultrawork/agents/plan.toml +2 -2
- package/packages/omo-codex/plugin/components/ultrawork/directive.md +3 -2
- package/packages/omo-codex/plugin/components/ultrawork/hooks/hooks.json +1 -1
- package/packages/omo-codex/plugin/components/ultrawork/package.json +1 -1
- package/packages/omo-codex/plugin/components/ultrawork/skills/ultrawork/SKILL.md +3 -2
- package/packages/omo-codex/plugin/components/ultrawork/skills/ulw-plan/SKILL.md +2 -2
- package/packages/omo-codex/plugin/components/ultrawork/skills/ulw-plan/references/full-workflow.md +3 -3
- package/packages/omo-codex/plugin/components/ultrawork/skills/ulw-plan/scripts/scaffold-plan.mjs +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/AGENTS.md +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/CHANGELOG.md +2 -0
- package/packages/omo-codex/plugin/components/ulw-loop/README.md +11 -11
- package/packages/omo-codex/plugin/components/ulw-loop/directive.md +3 -2
- package/packages/omo-codex/plugin/components/ulw-loop/dist/checkpoint-reconciliation.js +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/dist/cli-output.d.ts +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/dist/cli-output.js +9 -9
- package/packages/omo-codex/plugin/components/ulw-loop/dist/cli-steering.js +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/dist/cli.js +66 -66
- package/packages/omo-codex/plugin/components/ulw-loop/dist/codex-goal-instruction.js +4 -4
- package/packages/omo-codex/plugin/components/ulw-loop/dist/codex-hook.js +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/dist/plan-crud.js +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/dist/plan-io.js +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/dist/steering.js +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/dist/stop-resume-hook.js +2 -2
- package/packages/omo-codex/plugin/components/ulw-loop/hooks/hooks.json +4 -4
- package/packages/omo-codex/plugin/components/ulw-loop/package.json +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/skills/ulw-loop/SKILL.md +3 -3
- package/packages/omo-codex/plugin/components/ulw-loop/skills/ulw-loop/references/full-workflow.md +22 -25
- package/packages/omo-codex/plugin/components/ulw-loop/src/checkpoint-reconciliation.ts +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/src/cli-output.ts +9 -9
- package/packages/omo-codex/plugin/components/ulw-loop/src/cli-steering.ts +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/src/cli.ts +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/src/codex-goal-instruction.ts +4 -4
- package/packages/omo-codex/plugin/components/ulw-loop/src/codex-hook.ts +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/src/plan-crud.ts +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/src/plan-io.ts +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/src/steering.ts +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/src/stop-resume-hook.ts +2 -2
- package/packages/omo-codex/plugin/components/ulw-loop/test/cli-commands.test.ts +2 -2
- package/packages/omo-codex/plugin/components/ulw-loop/test/cli-entrypoint.test.ts +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/test/cli-helpers.test.ts +2 -2
- package/packages/omo-codex/plugin/components/ulw-loop/test/cli-steering-kind-guidance.test.ts +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/test/codex-hook.test.ts +2 -2
- package/packages/omo-codex/plugin/components/ulw-loop/test/fixtures/quality-gate-builder.ts +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/test/package-smoke.test.ts +5 -5
- package/packages/omo-codex/plugin/components/ulw-loop/test/plan-io.test.ts +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/test/quality-gate-roles.test.ts +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/test/steering.test.ts +1 -1
- package/packages/omo-codex/plugin/components/ulw-loop/test/stop-resume-hook.test.ts +1 -1
- package/packages/omo-codex/plugin/hooks/post-compact-resetting-git-bash-mcp-reminder.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-compact-resetting-lsp-diagnostics-cache.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-compact-resetting-project-rule-cache.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-tool-use-checking-codegraph-init-guidance.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-tool-use-checking-comments.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-tool-use-checking-lsp-diagnostics.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-tool-use-checking-thread-title-hygiene.json +1 -1
- package/packages/omo-codex/plugin/hooks/post-tool-use-matching-project-rules.json +1 -1
- package/packages/omo-codex/plugin/hooks/pre-tool-use-enforcing-unlimited-goal-budget.json +1 -1
- package/packages/omo-codex/plugin/hooks/pre-tool-use-guarding-ulw-loop-spawns.json +1 -1
- package/packages/omo-codex/plugin/hooks/pre-tool-use-recommending-git-bash-mcp.json +1 -1
- package/packages/omo-codex/plugin/hooks/session-start-checking-auto-update.json +1 -1
- package/packages/omo-codex/plugin/hooks/session-start-checking-bootstrap-provisioning.json +1 -1
- package/packages/omo-codex/plugin/hooks/session-start-checking-codegraph-bootstrap.json +1 -1
- package/packages/omo-codex/plugin/hooks/session-start-loading-project-rules.json +1 -1
- package/packages/omo-codex/plugin/hooks/session-start-recording-session-telemetry.json +1 -1
- package/packages/omo-codex/plugin/hooks/stop-checking-start-work-continuation.json +1 -1
- package/packages/omo-codex/plugin/hooks/stop-checking-ulw-loop-resume.json +1 -1
- package/packages/omo-codex/plugin/hooks/subagent-stop-checking-start-work-continuation.json +1 -1
- package/packages/omo-codex/plugin/hooks/subagent-stop-verifying-lazycodex-executor-evidence.json +1 -1
- package/packages/omo-codex/plugin/hooks/user-prompt-submit-checking-ultrawork-trigger.json +1 -1
- package/packages/omo-codex/plugin/hooks/user-prompt-submit-checking-ulw-loop-steering.json +1 -1
- package/packages/omo-codex/plugin/hooks/user-prompt-submit-loading-project-rules.json +1 -1
- package/packages/omo-codex/plugin/package-lock.json +13 -13
- package/packages/omo-codex/plugin/package.json +1 -1
- package/packages/omo-codex/plugin/scripts/sync-skills.mjs +8 -2
- package/packages/omo-codex/plugin/skills/ast-grep/SOURCE +1 -1
- package/packages/omo-codex/plugin/skills/ast-grep/install.ps1 +2 -2
- package/packages/omo-codex/plugin/skills/ast-grep/install.sh +1 -1
- package/packages/omo-codex/plugin/skills/ast-grep/references/install.md +2 -2
- package/packages/omo-codex/plugin/skills/ast-grep/tests/smoke.sh +1 -1
- package/packages/omo-codex/plugin/skills/coding-agent-sessions/SKILL.md +4 -3
- package/packages/omo-codex/plugin/skills/coding-agent-sessions/references/all-platforms.md +3 -1
- package/packages/omo-codex/plugin/skills/coding-agent-sessions/scripts/agent_sessions/aside_scanner.py +140 -0
- package/packages/omo-codex/plugin/skills/coding-agent-sessions/scripts/agent_sessions/scanners.py +3 -0
- package/packages/omo-codex/plugin/skills/data-scientist/SKILL.md +243 -0
- package/packages/omo-codex/plugin/skills/data-scientist/agents/openai.yaml +2 -0
- package/packages/omo-codex/plugin/skills/data-scientist/references/common-scenarios.md +176 -0
- package/packages/omo-codex/plugin/skills/data-scientist/references/execution-templates.md +197 -0
- package/packages/omo-codex/plugin/skills/data-scientist/references/integration-patterns.md +153 -0
- package/packages/omo-codex/plugin/skills/data-scientist/references/performance-benchmarks.md +37 -0
- package/packages/omo-codex/plugin/skills/data-scientist/references/uv-setup.md +78 -0
- package/packages/omo-codex/plugin/skills/data-scientist/scripts/quick-query.py +111 -0
- package/packages/omo-codex/plugin/skills/data-scientist/scripts/setup-uv.ps1 +53 -0
- package/packages/omo-codex/plugin/skills/data-scientist/scripts/setup-uv.sh +60 -0
- package/packages/omo-codex/plugin/skills/debugging/SKILL.md +1 -1
- package/packages/omo-codex/plugin/skills/programming/SKILL.md +1 -2
- package/packages/omo-codex/plugin/skills/start-work/SKILL.md +54 -9
- package/packages/omo-codex/plugin/skills/ultimate-browsing/SKILL.md +2 -2
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/__main__.py +8 -1
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/bias_check.py +11 -0
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/fetch_chain.py +90 -52
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/result_schema.py +10 -1
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/surrogate.py +214 -0
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/surrogates.yaml +60 -0
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/fixtures/amp_redirect_stub.html +7 -0
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/fixtures/search_interstitial.html +19 -0
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/fixtures/wayback_available.json +1 -0
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/fixtures/wayback_snapshot.html +1128 -0
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/test_surrogate.py +252 -0
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/test_surrogate_validators.py +78 -0
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/validators.py +46 -0
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/waf_detector.py +1 -1
- package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/waf_profiles.yaml +10 -5
- package/packages/omo-codex/plugin/skills/ultimate-browsing/references/agent-reach/social.md +1 -1
- package/packages/omo-codex/plugin/skills/ultimate-browsing/references/chrome-stealth.md +13 -11
- package/packages/omo-codex/plugin/skills/ultimate-browsing/references/insane-search/README.md +4 -4
- package/packages/omo-codex/plugin/skills/ultimate-browsing/references/insane-search/cache-archive.md +51 -50
- package/packages/omo-codex/plugin/skills/ultimate-browsing/references/insane-search/fallback.md +1 -1
- package/packages/omo-codex/plugin/skills/ultimate-browsing/references/insane-search/jina.md +8 -2
- package/packages/omo-codex/plugin/skills/ultimate-browsing/references/insane-search/naver.md +1 -1
- package/packages/omo-codex/plugin/skills/ultimate-browsing/references/insane-search/twitter.md +3 -3
- package/packages/omo-codex/plugin/skills/ultrawork/SKILL.md +3 -2
- package/packages/omo-codex/plugin/skills/ulw-loop/SKILL.md +3 -3
- package/packages/omo-codex/plugin/skills/ulw-loop/references/full-workflow.md +22 -25
- package/packages/omo-codex/plugin/skills/ulw-plan/SKILL.md +2 -2
- package/packages/omo-codex/plugin/skills/ulw-plan/references/full-workflow.md +3 -3
- package/packages/omo-codex/plugin/skills/ulw-plan/scripts/scaffold-plan.mjs +1 -1
- package/packages/omo-codex/plugin/skills/ulw-research/SKILL.md +127 -12
- package/packages/omo-codex/plugin/test/bootstrap-binlinks.test.mjs +12 -12
- package/packages/omo-codex/plugin/test/bootstrap-orchestration.test.mjs +36 -4
- package/packages/omo-codex/plugin/test/sync-skills-orchestration.test.mjs +1 -1
- package/packages/omo-codex/plugin/test/sync-skills-test-support.mjs +9 -2
- package/packages/omo-codex/plugin/test/sync-skills.test.mjs +12 -0
- package/packages/omo-codex/scripts/install-bin-links.test.mjs +56 -2
- package/packages/omo-codex/scripts/install-delegated-command.test.mjs +6 -6
- package/packages/omo-codex/scripts/install-dist/install-local.mjs +138 -65
- package/packages/omo-codex/scripts/install-local-entrypoint.test.mjs +4 -4
- package/packages/omo-codex/scripts/install-local.test.mjs +5 -2
- package/packages/shared-skills/index.mjs +19 -1
- package/packages/shared-skills/skills/ast-grep/SOURCE +1 -1
- package/packages/shared-skills/skills/ast-grep/install.ps1 +2 -2
- package/packages/shared-skills/skills/ast-grep/install.sh +1 -1
- package/packages/shared-skills/skills/ast-grep/references/install.md +2 -2
- package/packages/shared-skills/skills/ast-grep/tests/smoke.sh +1 -1
- package/packages/shared-skills/skills/coding-agent-sessions/SKILL.md +4 -3
- package/packages/shared-skills/skills/coding-agent-sessions/references/all-platforms.md +3 -1
- package/packages/shared-skills/skills/coding-agent-sessions/scripts/agent_sessions/aside_scanner.py +140 -0
- package/packages/shared-skills/skills/coding-agent-sessions/scripts/agent_sessions/scanners.py +3 -0
- package/packages/shared-skills/skills/data-scientist/SKILL.md +243 -0
- package/packages/shared-skills/skills/data-scientist/references/common-scenarios.md +176 -0
- package/packages/shared-skills/skills/data-scientist/references/execution-templates.md +197 -0
- package/packages/shared-skills/skills/data-scientist/references/integration-patterns.md +153 -0
- package/packages/shared-skills/skills/data-scientist/references/performance-benchmarks.md +37 -0
- package/packages/shared-skills/skills/data-scientist/references/uv-setup.md +78 -0
- package/packages/shared-skills/skills/data-scientist/scripts/quick-query.py +111 -0
- package/packages/shared-skills/skills/data-scientist/scripts/setup-uv.ps1 +53 -0
- package/packages/shared-skills/skills/data-scientist/scripts/setup-uv.sh +60 -0
- package/packages/shared-skills/skills/debugging/SKILL.md +1 -1
- package/packages/shared-skills/skills/programming/SKILL.md +1 -2
- package/packages/shared-skills/skills/start-work/SKILL.md +54 -9
- package/packages/shared-skills/skills/ultimate-browsing/SKILL.md +2 -2
- package/packages/shared-skills/skills/ultimate-browsing/engine/__main__.py +8 -1
- package/packages/shared-skills/skills/ultimate-browsing/engine/bias_check.py +11 -0
- package/packages/shared-skills/skills/ultimate-browsing/engine/fetch_chain.py +90 -52
- package/packages/shared-skills/skills/ultimate-browsing/engine/result_schema.py +10 -1
- package/packages/shared-skills/skills/ultimate-browsing/engine/surrogate.py +214 -0
- package/packages/shared-skills/skills/ultimate-browsing/engine/surrogates.yaml +60 -0
- package/packages/shared-skills/skills/ultimate-browsing/engine/tests/fixtures/amp_redirect_stub.html +7 -0
- package/packages/shared-skills/skills/ultimate-browsing/engine/tests/fixtures/search_interstitial.html +19 -0
- package/packages/shared-skills/skills/ultimate-browsing/engine/tests/fixtures/wayback_available.json +1 -0
- package/packages/shared-skills/skills/ultimate-browsing/engine/tests/fixtures/wayback_snapshot.html +1128 -0
- package/packages/shared-skills/skills/ultimate-browsing/engine/tests/test_surrogate.py +252 -0
- package/packages/shared-skills/skills/ultimate-browsing/engine/tests/test_surrogate_validators.py +78 -0
- package/packages/shared-skills/skills/ultimate-browsing/engine/validators.py +46 -0
- package/packages/shared-skills/skills/ultimate-browsing/engine/waf_detector.py +1 -1
- package/packages/shared-skills/skills/ultimate-browsing/engine/waf_profiles.yaml +10 -5
- package/packages/shared-skills/skills/ultimate-browsing/references/agent-reach/social.md +1 -1
- package/packages/shared-skills/skills/ultimate-browsing/references/chrome-stealth.md +13 -11
- package/packages/shared-skills/skills/ultimate-browsing/references/insane-search/README.md +4 -4
- package/packages/shared-skills/skills/ultimate-browsing/references/insane-search/cache-archive.md +51 -50
- package/packages/shared-skills/skills/ultimate-browsing/references/insane-search/fallback.md +1 -1
- package/packages/shared-skills/skills/ultimate-browsing/references/insane-search/jina.md +8 -2
- package/packages/shared-skills/skills/ultimate-browsing/references/insane-search/naver.md +1 -1
- package/packages/shared-skills/skills/ultimate-browsing/references/insane-search/twitter.md +3 -3
- package/packages/shared-skills/skills/ulw-plan/SKILL.md +2 -2
- package/packages/shared-skills/skills/ulw-plan/references/full-workflow.md +3 -3
- package/packages/shared-skills/skills/ulw-plan/scripts/scaffold-plan.mjs +1 -1
- package/packages/shared-skills/skills/ulw-research/SKILL.md +128 -12
- package/postinstall.mjs +6 -0
|
@@ -37,7 +37,7 @@ Read the matching reference before acting: [`references/insane-search/README.md`
|
|
|
37
37
|
## Tier 1 — insane-search (headless extraction)
|
|
38
38
|
|
|
39
39
|
**When**: content extraction, blocked-URL bypass, media metadata — no browser UI needed.
|
|
40
|
-
**Why first**: ~10x faster than a browser, no process spin-up; handles most "fetch this blocked page" requests via curl_cffi TLS impersonation, yt-dlp (1858 sites),
|
|
40
|
+
**Why first**: ~10x faster than a browser, no process spin-up; handles most "fetch this blocked page" requests via curl_cffi TLS impersonation, yt-dlp (1858 sites), official public APIs, mobile URL transforms, **Phase-2.5 surrogate archives** (Wayback / archive.today snapshots, provenance-tagged — see [`references/insane-search/cache-archive.md`](references/insane-search/cache-archive.md)), a key-gated Jina Reader (`JINA_API_KEY`), and a Playwright real-Chrome fallback. The engine lives **inside this skill** at `engine/` and is invoked as a module. Surrogate results are dated COPIES: a result whose `provenance` is `snapshot` must be reported with its `snapshot_timestamp`, never presented as the live page.
|
|
41
41
|
|
|
42
42
|
```bash
|
|
43
43
|
# Core command — auto-detects WAF, runs the full fetch grid (run from the skill dir):
|
|
@@ -60,7 +60,7 @@ The full engine harness (rules R1-R7, the Phase 0 official-API index, the no-sit
|
|
|
60
60
|
|
|
61
61
|
## Tier 1.5 — agent-reach (platform-native readers)
|
|
62
62
|
|
|
63
|
-
**When**: the target is a platform with a first-class API/CLI that beats generic fetching — especially Chinese platforms that stealth browsers still cannot reach cleanly. Several channels are zero-config (Douyin,
|
|
63
|
+
**When**: the target is a platform with a first-class API/CLI that beats generic fetching — especially Chinese platforms that stealth browsers still cannot reach cleanly. Several channels are zero-config (Douyin, V2EX, Reddit, RSS, YouTube); others need a one-time auth you supply via environment variables if you have access (`JINA_API_KEY` for Jina Reader — anonymous access is dead, see `references/insane-search/jina.md`; `TWITTER_*` for X; a transcription key for podcasts).
|
|
64
64
|
|
|
65
65
|
| Category | Platforms | Entry |
|
|
66
66
|
|---|---|---|
|
|
@@ -39,6 +39,10 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
39
39
|
help="Upper bound across all phases (default 12).")
|
|
40
40
|
p.add_argument("--no-playwright", action="store_true",
|
|
41
41
|
help="Skip Playwright fallback (curl-only).")
|
|
42
|
+
p.add_argument("--allow-proxy", action="store_true",
|
|
43
|
+
help=("Enable kind=proxy surrogate entries (raw relay routes; "
|
|
44
|
+
"MITM by construction — never citable alone, never "
|
|
45
|
+
"receives cookies/auth headers). Off by default."))
|
|
42
46
|
p.add_argument("--json", action="store_true",
|
|
43
47
|
help="Emit FetchResult as JSON to stdout (content omitted).")
|
|
44
48
|
p.add_argument("--trace", action="store_true",
|
|
@@ -56,6 +60,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
56
60
|
timeout=args.timeout,
|
|
57
61
|
max_attempts=args.max_attempts,
|
|
58
62
|
enable_playwright=not args.no_playwright,
|
|
63
|
+
allow_surrogate_proxy=args.allow_proxy,
|
|
59
64
|
)
|
|
60
65
|
except Exception as e:
|
|
61
66
|
print(f"engine fatal: {type(e).__name__}: {e}", file=sys.stderr)
|
|
@@ -96,7 +101,9 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
96
101
|
# Default: HTML to stdout, status to stderr.
|
|
97
102
|
print(result.content, end="")
|
|
98
103
|
print(f"\n[engine] ok={result.ok} verdict={result.verdict} "
|
|
99
|
-
f"profile={result.profile_used} attempts={len(result.trace)}"
|
|
104
|
+
f"profile={result.profile_used} attempts={len(result.trace)} "
|
|
105
|
+
f"provenance={result.provenance}"
|
|
106
|
+
+ (f" snapshot_ts={result.snapshot_timestamp}" if result.snapshot_timestamp else ""),
|
|
100
107
|
file=sys.stderr)
|
|
101
108
|
|
|
102
109
|
return 0 if result.ok else 1
|
|
@@ -54,6 +54,14 @@ URL_ALLOWLIST = {
|
|
|
54
54
|
"www.google.com", "google.com",
|
|
55
55
|
# Generic HTTP test endpoint for infrastructure / transport tests.
|
|
56
56
|
"httpbin.org",
|
|
57
|
+
# Surrogate-tier infrastructure (engine/surrogates.yaml): archives/relays
|
|
58
|
+
# that serve a copy of ANY target URL. Same no-site-preference category as
|
|
59
|
+
# the google.com Referer — they are not target sites.
|
|
60
|
+
"archive.org", "web.archive.org",
|
|
61
|
+
"r.jina.ai",
|
|
62
|
+
# Placeholder host in the proxy-entry shape example; "canonical non-routable
|
|
63
|
+
# doc domain" (.invalid TLD), can never be a real target.
|
|
64
|
+
"relay.invalid",
|
|
57
65
|
}
|
|
58
66
|
|
|
59
67
|
# Files / dirs that must be clean.
|
|
@@ -80,6 +88,9 @@ EXPLICIT_ALLOW_FILES = {
|
|
|
80
88
|
# None right now — add if needed with justification.
|
|
81
89
|
}
|
|
82
90
|
|
|
91
|
+
# NOTE-BIAS-OK convention for surrogate tests: fixture URLs (stub/interstitial
|
|
92
|
+
# captures from third-party services) carry the marker on the flagged line.
|
|
93
|
+
|
|
83
94
|
|
|
84
95
|
def _line_is_exempt(line: str, ext: str) -> bool:
|
|
85
96
|
markers = COMMENT_OK_MARKERS.get(ext, ())
|
|
@@ -40,6 +40,7 @@ def fetch(
|
|
|
40
40
|
timeout: int = 25,
|
|
41
41
|
max_attempts: int = 12,
|
|
42
42
|
enable_playwright: bool = True, # hook left for executor module
|
|
43
|
+
allow_surrogate_proxy: bool = False, # opt-in: kind=proxy registry entries
|
|
43
44
|
) -> FetchResult:
|
|
44
45
|
"""Fetch `url` using the generic grid.
|
|
45
46
|
|
|
@@ -175,73 +176,110 @@ def fetch(
|
|
|
175
176
|
if att.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
|
|
176
177
|
return _build_result(resp, att, trace, profile_used=profile_id)
|
|
177
178
|
|
|
178
|
-
# -------- Phase 3:
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
179
|
+
# -------- Phase 2.5 + 3: fallback ladder (profile-driven order) ---------
|
|
180
|
+
# Surrogate routes are NOT browser work, so they run even when the caller
|
|
181
|
+
# disabled Playwright; only browser executors honour `enable_playwright`.
|
|
182
|
+
try:
|
|
183
|
+
# Honour profile's `fallback_when_challenge` list — iterate the
|
|
184
|
+
# caller-declared order instead of capability-inferred single pick.
|
|
185
|
+
fb_profile = load_profile(profile_used or "unknown_challenge", profiles=profiles)
|
|
186
|
+
fb_order = fb_profile.get("fallback_when_challenge") or ["playwright_real_chrome"]
|
|
187
|
+
pw_attempt = None
|
|
188
|
+
pw_content = ""
|
|
189
|
+
for fb_name in fb_order:
|
|
190
|
+
if fb_name == "curl_grid_exhaust":
|
|
191
|
+
# Already performed in Phase 2; nothing more to do here.
|
|
192
|
+
continue
|
|
193
|
+
if fb_name.startswith("surrogate_"):
|
|
194
|
+
# Phase 2.5: archive/reader/proxy routes serving a copy of
|
|
195
|
+
# the target. Cheaper than a browser; provenance-labeled.
|
|
196
|
+
from . import surrogate as surrogate_mod
|
|
197
|
+
s_atts, s_meta, s_content = surrogate_mod.run_surrogate(
|
|
193
198
|
url,
|
|
194
|
-
|
|
199
|
+
registry=surrogate_mod.load_surrogates(),
|
|
200
|
+
allow_proxy=allow_surrogate_proxy,
|
|
201
|
+
timeout=timeout,
|
|
195
202
|
success_selectors=success_selectors,
|
|
196
|
-
device_class=device_class,
|
|
197
|
-
force_executor=fb_name,
|
|
198
203
|
)
|
|
199
|
-
trace.
|
|
200
|
-
|
|
204
|
+
trace.extend(s_atts)
|
|
205
|
+
s_att = s_atts[-1]
|
|
206
|
+
if s_att.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
|
|
201
207
|
return FetchResult(
|
|
202
208
|
ok=True,
|
|
203
|
-
content=
|
|
204
|
-
final_url=
|
|
205
|
-
verdict=
|
|
209
|
+
content=s_content,
|
|
210
|
+
final_url=s_att.url,
|
|
211
|
+
verdict=s_att.verdict,
|
|
206
212
|
profile_used=profile_used,
|
|
207
213
|
trace=trace,
|
|
208
|
-
summary=
|
|
214
|
+
summary=(
|
|
215
|
+
f"surrogate {s_meta.get('surrogate')} succeeded"
|
|
216
|
+
f" (provenance={s_meta.get('provenance')}"
|
|
217
|
+
+ (f" snapshot_ts={s_meta.get('snapshot_timestamp')})" if s_meta.get('snapshot_timestamp') else ")")
|
|
218
|
+
),
|
|
219
|
+
provenance=str(s_meta.get("provenance", "live")),
|
|
220
|
+
snapshot_timestamp=s_meta.get("snapshot_timestamp"),
|
|
221
|
+
trust=str(s_meta.get("trust", "origin")),
|
|
209
222
|
)
|
|
210
|
-
|
|
211
|
-
if
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
223
|
+
continue
|
|
224
|
+
if not enable_playwright:
|
|
225
|
+
continue
|
|
226
|
+
from .executor import run_playwright_fallback # lazy import
|
|
227
|
+
pw_attempt, pw_content = run_playwright_fallback(
|
|
228
|
+
url,
|
|
229
|
+
profile_id=profile_used or "unknown_challenge",
|
|
230
|
+
success_selectors=success_selectors,
|
|
231
|
+
device_class=device_class,
|
|
232
|
+
force_executor=fb_name,
|
|
233
|
+
)
|
|
234
|
+
trace.append(pw_attempt)
|
|
235
|
+
if pw_attempt.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
|
|
236
|
+
return FetchResult(
|
|
237
|
+
ok=True,
|
|
238
|
+
content=pw_content,
|
|
239
|
+
final_url=pw_attempt.url,
|
|
240
|
+
verdict=pw_attempt.verdict,
|
|
241
|
+
profile_used=profile_used,
|
|
242
|
+
trace=trace,
|
|
243
|
+
summary=f"Playwright fallback succeeded via {fb_name}",
|
|
221
244
|
)
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
245
|
+
# Synthesize a placeholder only when the profile genuinely offers no
|
|
246
|
+
# fallback route. Entries skipped because the caller disabled the
|
|
247
|
+
# browser are a caller choice, not a profile defect — no trace noise.
|
|
248
|
+
actionable = [n for n in fb_order if n != "curl_grid_exhaust"]
|
|
249
|
+
if pw_attempt is None and not actionable:
|
|
250
|
+
pw_attempt = Attempt(
|
|
225
251
|
phase="fallback",
|
|
226
|
-
executor="
|
|
252
|
+
executor="none",
|
|
227
253
|
url=url,
|
|
228
254
|
url_transform="original",
|
|
229
255
|
impersonate=None,
|
|
230
256
|
referer="",
|
|
231
257
|
verdict=Verdict.UNKNOWN.value,
|
|
232
|
-
error="
|
|
233
|
-
)
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
258
|
+
error="profile has empty fallback_when_challenge",
|
|
259
|
+
)
|
|
260
|
+
trace.append(pw_attempt)
|
|
261
|
+
except ImportError:
|
|
262
|
+
trace.append(Attempt(
|
|
263
|
+
phase="fallback",
|
|
264
|
+
executor="playwright",
|
|
265
|
+
url=url,
|
|
266
|
+
url_transform="original",
|
|
267
|
+
impersonate=None,
|
|
268
|
+
referer="",
|
|
269
|
+
verdict=Verdict.UNKNOWN.value,
|
|
270
|
+
error="executor module not available",
|
|
271
|
+
))
|
|
272
|
+
except (RuntimeError, OSError) as e:
|
|
273
|
+
trace.append(Attempt(
|
|
274
|
+
phase="fallback",
|
|
275
|
+
executor="playwright",
|
|
276
|
+
url=url,
|
|
277
|
+
url_transform="original",
|
|
278
|
+
impersonate=None,
|
|
279
|
+
referer="",
|
|
280
|
+
verdict=Verdict.UNKNOWN.value,
|
|
281
|
+
error=f"{type(e).__name__}:{str(e)[:200]}",
|
|
282
|
+
))
|
|
245
283
|
|
|
246
284
|
# -------- Give up, return best we have ----------------------------------
|
|
247
285
|
summary = format_summary(trace, profile_used)
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from dataclasses import asdict, dataclass, field
|
|
4
|
-
from typing import Optional
|
|
4
|
+
from typing import Literal, Optional
|
|
5
|
+
|
|
6
|
+
Provenance = Literal["live", "snapshot", "proxy"]
|
|
7
|
+
Trust = Literal["origin", "archive", "untrusted"]
|
|
5
8
|
|
|
6
9
|
|
|
7
10
|
@dataclass
|
|
@@ -32,6 +35,9 @@ class FetchResult:
|
|
|
32
35
|
profile_used: Optional[str] = None
|
|
33
36
|
trace: list[Attempt] = field(default_factory=list)
|
|
34
37
|
summary: str = ""
|
|
38
|
+
provenance: str = "live" # live | snapshot | proxy
|
|
39
|
+
snapshot_timestamp: Optional[str] = None # archive's own timestamp, when snapshot
|
|
40
|
+
trust: str = "origin" # origin | archive | untrusted
|
|
35
41
|
|
|
36
42
|
def to_dict(self) -> dict:
|
|
37
43
|
return {
|
|
@@ -41,5 +47,8 @@ class FetchResult:
|
|
|
41
47
|
"profile_used": self.profile_used,
|
|
42
48
|
"trace": [a.to_dict() for a in self.trace],
|
|
43
49
|
"summary": self.summary,
|
|
50
|
+
"provenance": self.provenance,
|
|
51
|
+
"snapshot_timestamp": self.snapshot_timestamp,
|
|
52
|
+
"trust": self.trust,
|
|
44
53
|
"content_length": len(self.content),
|
|
45
54
|
}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""Phase 2.5 surrogate retrieval: archive / reader / proxy routes for blocked origins.
|
|
2
|
+
|
|
3
|
+
When the curl grid cannot fetch the live page, this module reads a declarative
|
|
4
|
+
registry (engine/surrogates.yaml) and tries third-party routes that serve a
|
|
5
|
+
copy of the target instead. Provenance semantics are contractual:
|
|
6
|
+
|
|
7
|
+
kind=archive -> provenance="snapshot" (+ archive's own timestamp), trust=archive
|
|
8
|
+
kind=reader -> provenance="live" (server-side re-render of the live page)
|
|
9
|
+
kind=proxy -> provenance="proxy", trust=untrusted; requires allow_proxy=True
|
|
10
|
+
and never sends Cookie/Authorization headers (relay = MITM)
|
|
11
|
+
|
|
12
|
+
Every response body passes through validators.validate() with target_url set,
|
|
13
|
+
so AMP-style redirect stubs and search-engine interstitials are rejected
|
|
14
|
+
rather than recorded as wins. Entries whose `last_verified` is older than
|
|
15
|
+
MAX_STALE_DAYS are tried after fresh ones (routes rot faster than they are
|
|
16
|
+
maintained).
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import time
|
|
23
|
+
from datetime import date
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
import yaml
|
|
27
|
+
except ImportError:
|
|
28
|
+
yaml = None
|
|
29
|
+
|
|
30
|
+
from .result_schema import Attempt
|
|
31
|
+
from .validators import Verdict, validate
|
|
32
|
+
|
|
33
|
+
SURROGATES_PATH = os.path.join(os.path.dirname(__file__), "surrogates.yaml")
|
|
34
|
+
MAX_STALE_DAYS = 90
|
|
35
|
+
|
|
36
|
+
DEFAULT_SURROGATES: dict = {
|
|
37
|
+
"wayback": {
|
|
38
|
+
"kind": "archive",
|
|
39
|
+
"trust": "archive",
|
|
40
|
+
"enabled": True,
|
|
41
|
+
"last_verified": "2026-08-09",
|
|
42
|
+
"discovery": {
|
|
43
|
+
"url": "https://archive.org/wayback/available?url={target}",
|
|
44
|
+
"flag_pointer": "archived_snapshots.closest.available",
|
|
45
|
+
"snapshot_url_pointer": "archived_snapshots.closest.url",
|
|
46
|
+
"snapshot_timestamp_pointer": "archived_snapshots.closest.timestamp",
|
|
47
|
+
},
|
|
48
|
+
"min_body_bytes": 3072,
|
|
49
|
+
},
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def load_surrogates(path: str = SURROGATES_PATH) -> dict:
|
|
54
|
+
if yaml is None:
|
|
55
|
+
return dict(DEFAULT_SURROGATES)
|
|
56
|
+
try:
|
|
57
|
+
with open(path, encoding="utf-8") as fh:
|
|
58
|
+
loaded = yaml.safe_load(fh) or {}
|
|
59
|
+
except (OSError, yaml.YAMLError):
|
|
60
|
+
return dict(DEFAULT_SURROGATES)
|
|
61
|
+
if not isinstance(loaded, dict):
|
|
62
|
+
return dict(DEFAULT_SURROGATES)
|
|
63
|
+
usable = {k: v for k, v in loaded.items() if isinstance(v, dict) and k}
|
|
64
|
+
return usable or dict(DEFAULT_SURROGATES)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def is_stale(entry: dict, max_age_days: int = MAX_STALE_DAYS, today: date | None = None) -> bool:
|
|
68
|
+
verified = entry.get("last_verified")
|
|
69
|
+
if isinstance(verified, date):
|
|
70
|
+
parsed = verified
|
|
71
|
+
elif isinstance(verified, str):
|
|
72
|
+
try:
|
|
73
|
+
parsed = date.fromisoformat(verified)
|
|
74
|
+
except ValueError:
|
|
75
|
+
return True
|
|
76
|
+
else:
|
|
77
|
+
return True
|
|
78
|
+
return ((today or date.today()) - parsed).days > max_age_days
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _http_get(url: str, *, headers: dict | None = None, timeout: int = 25):
|
|
82
|
+
try:
|
|
83
|
+
from curl_cffi import requests as cffi_requests
|
|
84
|
+
except ImportError as exc:
|
|
85
|
+
raise RuntimeError("curl_cffi not installed") from exc
|
|
86
|
+
return cffi_requests.get(
|
|
87
|
+
url,
|
|
88
|
+
impersonate="chrome",
|
|
89
|
+
headers=headers or {"Accept": "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8"},
|
|
90
|
+
timeout=timeout,
|
|
91
|
+
allow_redirects=True,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _json_pointer(obj, dotted: str):
|
|
96
|
+
current = obj
|
|
97
|
+
for part in dotted.split("."):
|
|
98
|
+
if not isinstance(current, dict):
|
|
99
|
+
return None
|
|
100
|
+
current = current.get(part)
|
|
101
|
+
return current
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _provenance_of(kind: str) -> str:
|
|
105
|
+
if kind == "archive":
|
|
106
|
+
return "snapshot"
|
|
107
|
+
if kind == "proxy":
|
|
108
|
+
return "proxy"
|
|
109
|
+
return "live"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _skip_reason(name: str, entry: dict, allow_proxy: bool) -> str | None:
|
|
113
|
+
kind = entry.get("kind", "archive")
|
|
114
|
+
if kind == "proxy" and not allow_proxy:
|
|
115
|
+
return "proxy_requires_allow_proxy_flag"
|
|
116
|
+
required_env = entry.get("enabled_env")
|
|
117
|
+
if required_env and not os.environ.get(str(required_env)):
|
|
118
|
+
return f"missing_env:{required_env}"
|
|
119
|
+
if entry.get("enabled") is False and kind != "proxy":
|
|
120
|
+
return "disabled"
|
|
121
|
+
return None
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _attempt(name: str, url: str, verdict: Verdict, error=None, status: int = 0, size: int = 0, elapsed: float = 0.0) -> Attempt:
|
|
125
|
+
return Attempt(
|
|
126
|
+
phase="surrogate",
|
|
127
|
+
executor=f"surrogate_{name}",
|
|
128
|
+
url=url,
|
|
129
|
+
url_transform="original",
|
|
130
|
+
impersonate=None,
|
|
131
|
+
referer="",
|
|
132
|
+
status=status,
|
|
133
|
+
body_size=size,
|
|
134
|
+
verdict=verdict.value,
|
|
135
|
+
error=(str(error)[:200] if error else None),
|
|
136
|
+
elapsed_s=round(elapsed, 3),
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _resolve_target(name: str, entry: dict, target: str, timeout: int) -> tuple[str | None, str | None]:
|
|
141
|
+
discovery = entry.get("discovery")
|
|
142
|
+
if not discovery:
|
|
143
|
+
host_rotation = entry.get("host_rotation") or [None]
|
|
144
|
+
host = host_rotation[0]
|
|
145
|
+
template = entry.get("fetch")
|
|
146
|
+
if not template or host is None and "{host}" in template:
|
|
147
|
+
return None, None
|
|
148
|
+
return template.format(host=host, target=target), None
|
|
149
|
+
discovery_url = discovery.get("url", "").format(target=target)
|
|
150
|
+
resp = _http_get(discovery_url, timeout=timeout)
|
|
151
|
+
payload = resp.json()
|
|
152
|
+
if discovery.get("flag_pointer") and not _json_pointer(payload, discovery["flag_pointer"]):
|
|
153
|
+
return None, None
|
|
154
|
+
snap_url = _json_pointer(payload, discovery.get("snapshot_url_pointer", ""))
|
|
155
|
+
timestamp = _json_pointer(payload, discovery.get("snapshot_timestamp_pointer", "")) or None
|
|
156
|
+
return snap_url, timestamp
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def run_surrogate(
|
|
160
|
+
target: str,
|
|
161
|
+
*,
|
|
162
|
+
registry: dict | None = None,
|
|
163
|
+
allow_proxy: bool = False,
|
|
164
|
+
timeout: int = 25,
|
|
165
|
+
success_selectors: list[str] | None = None,
|
|
166
|
+
) -> tuple[list[Attempt], dict, str]:
|
|
167
|
+
registry = dict(registry) if registry else load_surrogates()
|
|
168
|
+
ordered = sorted(registry.items(), key=lambda kv: int(is_stale(kv[1])))
|
|
169
|
+
live_meta = {"provenance": "live", "snapshot_timestamp": None, "trust": "origin", "surrogate": None}
|
|
170
|
+
attempts: list[Attempt] = []
|
|
171
|
+
|
|
172
|
+
for name, entry in ordered:
|
|
173
|
+
kind = entry.get("kind", "archive")
|
|
174
|
+
reason = _skip_reason(name, entry, allow_proxy)
|
|
175
|
+
if reason:
|
|
176
|
+
attempts.append(_attempt(name, target, Verdict.UNKNOWN, error=f"skipped:{reason}"))
|
|
177
|
+
continue
|
|
178
|
+
started = time.time()
|
|
179
|
+
try:
|
|
180
|
+
fetch_url, timestamp = _resolve_target(name, entry, target, timeout)
|
|
181
|
+
except Exception as exc:
|
|
182
|
+
attempts.append(_attempt(name, target, Verdict.UNKNOWN, error=f"{type(exc).__name__}:{exc}", elapsed=time.time() - started))
|
|
183
|
+
continue
|
|
184
|
+
if not fetch_url:
|
|
185
|
+
attempts.append(_attempt(name, target, Verdict.UNKNOWN, error="no snapshot available", elapsed=time.time() - started))
|
|
186
|
+
continue
|
|
187
|
+
try:
|
|
188
|
+
resp = _http_get(fetch_url, timeout=timeout)
|
|
189
|
+
except Exception as exc:
|
|
190
|
+
attempts.append(_attempt(name, fetch_url, Verdict.UNKNOWN, error=f"{type(exc).__name__}:{exc}", elapsed=time.time() - started))
|
|
191
|
+
continue
|
|
192
|
+
body = getattr(resp, "text", "") or ""
|
|
193
|
+
status = int(getattr(resp, "status_code", 0) or 0)
|
|
194
|
+
min_bytes = int(entry.get("min_body_bytes", 3072))
|
|
195
|
+
att = _attempt(name, fetch_url, Verdict.UNKNOWN, status=status, size=len(body), elapsed=time.time() - started)
|
|
196
|
+
attempts.append(att)
|
|
197
|
+
if status >= 400 or len(body) < min_bytes:
|
|
198
|
+
att.error = f"surrogate_unusable:status={status},size={len(body)}"
|
|
199
|
+
continue
|
|
200
|
+
vr = validate(resp, success_selectors=success_selectors, target_url=target)
|
|
201
|
+
att.verdict = vr.verdict.value
|
|
202
|
+
att.reasons = vr.reasons
|
|
203
|
+
if vr.ok:
|
|
204
|
+
meta = {
|
|
205
|
+
"provenance": _provenance_of(kind),
|
|
206
|
+
"snapshot_timestamp": timestamp if kind == "archive" else None,
|
|
207
|
+
"trust": str(entry.get("trust", "archive")),
|
|
208
|
+
"surrogate": name,
|
|
209
|
+
}
|
|
210
|
+
return attempts, meta, body
|
|
211
|
+
|
|
212
|
+
if not attempts:
|
|
213
|
+
attempts.append(_attempt("none", target, Verdict.UNKNOWN, error="no surrogate entries"))
|
|
214
|
+
return attempts, live_meta, ""
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Surrogate routes — third-party services serving a COPY of a page when the
|
|
2
|
+
# live origin blocks the fetch chain (Phase 2.5, between grid and Playwright).
|
|
3
|
+
#
|
|
4
|
+
# Rules (same discipline as waf_profiles.yaml):
|
|
5
|
+
# * Site-agnostic infrastructure ONLY (No-Site-Name Rule): every host here
|
|
6
|
+
# must work for any target URL. No target-site domains, selectors, brand
|
|
7
|
+
# names.
|
|
8
|
+
# * Surrogate routes ROT — the 2026-08 probe found 4 of 6 known routes dead
|
|
9
|
+
# or returning stubs. Entries must carry `last_verified` (ISO date) and a
|
|
10
|
+
# passing liveness record in the PR that adds them; refresh quarterly.
|
|
11
|
+
# Entries older than 90 days are tried after fresh ones and flagged.
|
|
12
|
+
# * kind semantics:
|
|
13
|
+
# archive — serves a dated snapshot; provenance=snapshot, trust=archive.
|
|
14
|
+
# reader — re-renders the LIVE page server-side; provenance=live.
|
|
15
|
+
# proxy — raw forwarding relay; MITM by construction, trust=untrusted,
|
|
16
|
+
# skipped unless the caller passes --allow-proxy, and never
|
|
17
|
+
# receives Cookie/Authorization headers.
|
|
18
|
+
# * keys:
|
|
19
|
+
# discovery.json_pointer — dotted path into the discovery JSON payload.
|
|
20
|
+
# host_rotation — try hostile-domain cousins in order (one may be
|
|
21
|
+
# blocked while another answers).
|
|
22
|
+
# enabled_env — entry is inert until that env var is set (key-gated).
|
|
23
|
+
# min_body_bytes — response floor; below it the body is a stub/error.
|
|
24
|
+
wayback:
|
|
25
|
+
kind: archive
|
|
26
|
+
trust: archive
|
|
27
|
+
enabled: true
|
|
28
|
+
last_verified: 2026-08-09
|
|
29
|
+
discovery:
|
|
30
|
+
url: "https://archive.org/wayback/available?url={target}"
|
|
31
|
+
flag_pointer: "archived_snapshots.closest.available"
|
|
32
|
+
snapshot_url_pointer: "archived_snapshots.closest.url"
|
|
33
|
+
snapshot_timestamp_pointer: "archived_snapshots.closest.timestamp"
|
|
34
|
+
min_body_bytes: 3072
|
|
35
|
+
|
|
36
|
+
archive_today:
|
|
37
|
+
kind: archive
|
|
38
|
+
trust: archive
|
|
39
|
+
enabled: true
|
|
40
|
+
last_verified: 2026-08-09
|
|
41
|
+
host_rotation: ["archive.ph", "archive.md", "archive.li", "archive.is"]
|
|
42
|
+
fetch: "https://{host}/newest/{target}"
|
|
43
|
+
min_body_bytes: 3072
|
|
44
|
+
|
|
45
|
+
jina_reader:
|
|
46
|
+
kind: reader
|
|
47
|
+
trust: archive
|
|
48
|
+
enabled: true
|
|
49
|
+
enabled_env: JINA_API_KEY
|
|
50
|
+
last_verified: 2026-08-09
|
|
51
|
+
fetch: "https://r.jina.ai/{target}"
|
|
52
|
+
min_body_bytes: 512
|
|
53
|
+
|
|
54
|
+
anon_relay_example:
|
|
55
|
+
kind: proxy
|
|
56
|
+
trust: untrusted
|
|
57
|
+
enabled: false
|
|
58
|
+
last_verified: 2026-08-09
|
|
59
|
+
fetch: "https://relay.invalid/?u={target}"
|
|
60
|
+
min_body_bytes: 3072
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
<HTML><HEAD>
|
|
2
|
+
<meta http-equiv="content-type" content="text/html;charset=utf-8">
|
|
3
|
+
<TITLE>Redirecting</TITLE>
|
|
4
|
+
<META HTTP-EQUIV="refresh" content="0; url=https://en.wikipedia.org/wiki/Web_archiving">
|
|
5
|
+
</HEAD>
|
|
6
|
+
<BODY onLoad="location.replace('https://en.wikipedia.org/wiki/Web_archiving'+document.location.hash)">
|
|
7
|
+
</BODY></HTML>
|