oh-my-opencode 4.19.3 → 5.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (347) hide show
  1. package/.agents/command/get-unpublished-changes.md +2 -0
  2. package/.agents/command/omomomo.md +1 -1
  3. package/.agents/command/publish.md +7 -0
  4. package/.agents/skills/get-unpublished-changes/SKILL.md +2 -0
  5. package/.agents/skills/hyperplan/SKILL.md +3 -3
  6. package/.agents/skills/omomomo/SKILL.md +1 -1
  7. package/.agents/skills/publish/SKILL.md +7 -0
  8. package/.opencode/command/get-unpublished-changes.md +2 -0
  9. package/.opencode/command/omomomo.md +1 -1
  10. package/.opencode/command/publish.md +7 -0
  11. package/.opencode/skills/hyperplan/SKILL.md +3 -3
  12. package/README.ja.md +1 -1
  13. package/README.ko.md +1 -1
  14. package/README.md +6 -5
  15. package/README.ru.md +1 -1
  16. package/README.zh-cn.md +1 -1
  17. package/bin/oh-my-opencode.js +14 -1
  18. package/bin/oh-my-opencode.test.ts +21 -0
  19. package/dist/agents/sisyphus-junior/agent.d.ts +1 -1
  20. package/dist/cli/doctor/checks/deprecated-reasoning-keys.d.ts +2 -0
  21. package/dist/cli/index.js +2925 -1919
  22. package/dist/cli-node/index.js +2925 -1919
  23. package/dist/config/schema/agent-overrides.d.ts +1343 -15
  24. package/dist/config/schema/categories.d.ts +132 -0
  25. package/dist/config/schema/fallback-models.d.ts +50 -0
  26. package/dist/config/schema/oh-my-opencode-config.d.ts +1322 -11
  27. package/dist/config-migration/index.d.ts +1 -0
  28. package/dist/config-migration/migration-plans.d.ts +1 -0
  29. package/dist/config-migration/reasoning-unification.d.ts +3 -0
  30. package/dist/features/monitor/batcher.d.ts +3 -1
  31. package/dist/features/monitor/manager-internals.d.ts +1 -0
  32. package/dist/features/monitor/output-injector-types.d.ts +2 -0
  33. package/dist/features/monitor/output-injector.d.ts +6 -0
  34. package/dist/features/team-mode/tools/lifecycle-test-fixture.d.ts +2 -0
  35. package/dist/hooks/codegraph-bootstrap/command-runner.d.ts +1 -0
  36. package/dist/hooks/model-fallback/next-fallback.d.ts +1 -0
  37. package/dist/hooks/runtime-fallback/constants.d.ts +1 -1
  38. package/dist/hooks/todo-continuation-enforcer/types.d.ts +1 -0
  39. package/dist/hooks/todo-continuation-enforcer/unrecoverable-request-error.d.ts +9 -0
  40. package/dist/hooks/tool-pair-validator/hook.test-support.d.ts +29 -0
  41. package/dist/hooks/tool-pair-validator/tool-part-ids.d.ts +14 -5
  42. package/dist/hooks/tool-pair-validator/tool-result-repair.d.ts +4 -3
  43. package/dist/hooks/tool-pair-validator/types.d.ts +5 -22
  44. package/dist/index.js +4864 -4011
  45. package/dist/mcp/lsp.d.ts +1 -0
  46. package/dist/oh-my-opencode.schema.json +3221 -222
  47. package/dist/plugin-handlers/prometheus-agent-config-builder.d.ts +1 -0
  48. package/dist/shared/agent-variant.d.ts +11 -0
  49. package/dist/shared/session-prompt-params-helpers.d.ts +6 -1
  50. package/dist/shared/tmux/constants.d.ts +1 -1
  51. package/dist/skills/ast-grep/SOURCE +1 -1
  52. package/dist/skills/ast-grep/install.ps1 +2 -2
  53. package/dist/skills/ast-grep/install.sh +1 -1
  54. package/dist/skills/ast-grep/references/install.md +2 -2
  55. package/dist/skills/ast-grep/tests/smoke.sh +1 -1
  56. package/dist/skills/coding-agent-sessions/SKILL.md +4 -3
  57. package/dist/skills/coding-agent-sessions/references/all-platforms.md +3 -1
  58. package/dist/skills/coding-agent-sessions/scripts/agent_sessions/aside_scanner.py +140 -0
  59. package/dist/skills/coding-agent-sessions/scripts/agent_sessions/scanners.py +3 -0
  60. package/dist/skills/data-scientist/SKILL.md +243 -0
  61. package/dist/skills/data-scientist/references/common-scenarios.md +176 -0
  62. package/dist/skills/data-scientist/references/execution-templates.md +197 -0
  63. package/dist/skills/data-scientist/references/integration-patterns.md +153 -0
  64. package/dist/skills/data-scientist/references/performance-benchmarks.md +37 -0
  65. package/dist/skills/data-scientist/references/uv-setup.md +78 -0
  66. package/dist/skills/data-scientist/scripts/quick-query.py +111 -0
  67. package/dist/skills/data-scientist/scripts/setup-uv.ps1 +53 -0
  68. package/dist/skills/data-scientist/scripts/setup-uv.sh +60 -0
  69. package/dist/skills/debugging/SKILL.md +1 -1
  70. package/dist/skills/programming/SKILL.md +1 -2
  71. package/dist/skills/start-work/SKILL.md +54 -9
  72. package/dist/skills/ultimate-browsing/SKILL.md +2 -2
  73. package/dist/skills/ultimate-browsing/engine/__main__.py +8 -1
  74. package/dist/skills/ultimate-browsing/engine/bias_check.py +11 -0
  75. package/dist/skills/ultimate-browsing/engine/fetch_chain.py +90 -52
  76. package/dist/skills/ultimate-browsing/engine/result_schema.py +10 -1
  77. package/dist/skills/ultimate-browsing/engine/surrogate.py +214 -0
  78. package/dist/skills/ultimate-browsing/engine/surrogates.yaml +60 -0
  79. package/dist/skills/ultimate-browsing/engine/tests/fixtures/amp_redirect_stub.html +7 -0
  80. package/dist/skills/ultimate-browsing/engine/tests/fixtures/search_interstitial.html +19 -0
  81. package/dist/skills/ultimate-browsing/engine/tests/fixtures/wayback_available.json +1 -0
  82. package/dist/skills/ultimate-browsing/engine/tests/fixtures/wayback_snapshot.html +1128 -0
  83. package/dist/skills/ultimate-browsing/engine/tests/test_surrogate.py +252 -0
  84. package/dist/skills/ultimate-browsing/engine/tests/test_surrogate_validators.py +78 -0
  85. package/dist/skills/ultimate-browsing/engine/validators.py +46 -0
  86. package/dist/skills/ultimate-browsing/engine/waf_detector.py +1 -1
  87. package/dist/skills/ultimate-browsing/engine/waf_profiles.yaml +10 -5
  88. package/dist/skills/ultimate-browsing/references/agent-reach/social.md +1 -1
  89. package/dist/skills/ultimate-browsing/references/chrome-stealth.md +13 -11
  90. package/dist/skills/ultimate-browsing/references/insane-search/README.md +4 -4
  91. package/dist/skills/ultimate-browsing/references/insane-search/cache-archive.md +51 -50
  92. package/dist/skills/ultimate-browsing/references/insane-search/fallback.md +1 -1
  93. package/dist/skills/ultimate-browsing/references/insane-search/jina.md +8 -2
  94. package/dist/skills/ultimate-browsing/references/insane-search/naver.md +1 -1
  95. package/dist/skills/ultimate-browsing/references/insane-search/twitter.md +3 -3
  96. package/dist/skills/ulw-plan/SKILL.md +2 -2
  97. package/dist/skills/ulw-plan/references/full-workflow.md +3 -3
  98. package/dist/skills/ulw-plan/scripts/scaffold-plan.mjs +1 -1
  99. package/dist/skills/ulw-research/SKILL.md +128 -12
  100. package/dist/tools/delegate-task/builtin-categories.d.ts +1 -0
  101. package/dist/tools/delegate-task/builtin-category-definition.d.ts +1 -0
  102. package/dist/tools/delegate-task/constants.d.ts +1 -1
  103. package/dist/tui.js +1271 -1108
  104. package/docs/reference/web-terminal-visual-qa.md +1 -1
  105. package/package.json +27 -19
  106. package/packages/lsp-core/src/lsp/client-diagnostics-freshness.integration.test.ts +2 -2
  107. package/packages/lsp-core/src/lsp/connection.ts +1 -1
  108. package/packages/lsp-daemon/dist/cli.js +27 -13
  109. package/packages/lsp-daemon/dist/client.js +49 -35
  110. package/packages/lsp-daemon/dist/ensure-daemon.d.ts +1 -0
  111. package/packages/lsp-daemon/dist/ensure-daemon.js +18 -5
  112. package/packages/lsp-daemon/dist/index.js +34 -20
  113. package/packages/lsp-tools-mcp/dist/cli.js +1 -1
  114. package/packages/lsp-tools-mcp/dist/lsp/manager.js +1 -1
  115. package/packages/lsp-tools-mcp/dist/mcp.js +1 -1
  116. package/packages/lsp-tools-mcp/dist/tools.js +1 -1
  117. package/packages/omo-codex/plugin/.codex-plugin/plugin.json +1 -1
  118. package/packages/omo-codex/plugin/components/bootstrap/dist/cli.js +289 -95
  119. package/packages/omo-codex/plugin/components/bootstrap/hooks/hooks.json +1 -1
  120. package/packages/omo-codex/plugin/components/bootstrap/package.json +1 -1
  121. package/packages/omo-codex/plugin/components/bootstrap/src/setup.ts +7 -7
  122. package/packages/omo-codex/plugin/components/bootstrap/src/worker.ts +3 -0
  123. package/packages/omo-codex/plugin/components/codegraph/dist/cli.js +385 -64
  124. package/packages/omo-codex/plugin/components/codegraph/dist/serve.js +336 -47
  125. package/packages/omo-codex/plugin/components/codegraph/package.json +1 -1
  126. package/packages/omo-codex/plugin/components/codegraph/test/hook.test.ts +6 -0
  127. package/packages/omo-codex/plugin/components/comment-checker/hooks/hooks.json +1 -1
  128. package/packages/omo-codex/plugin/components/comment-checker/package.json +1 -1
  129. package/packages/omo-codex/plugin/components/git-bash/hooks/hooks.json +2 -2
  130. package/packages/omo-codex/plugin/components/git-bash/package.json +1 -1
  131. package/packages/omo-codex/plugin/components/lazycodex-executor-verify/hooks/hooks.json +1 -1
  132. package/packages/omo-codex/plugin/components/lazycodex-executor-verify/package.json +1 -1
  133. package/packages/omo-codex/plugin/components/lsp/dist/.omo-runtime-manifest.json +3 -3
  134. package/packages/omo-codex/plugin/components/lsp/dist/cli.js +56 -42
  135. package/packages/omo-codex/plugin/components/lsp/hooks/hooks.json +2 -2
  136. package/packages/omo-codex/plugin/components/lsp/package.json +1 -1
  137. package/packages/omo-codex/plugin/components/rules/dist/cli.js +1 -1
  138. package/packages/omo-codex/plugin/components/rules/hooks/hooks.json +4 -4
  139. package/packages/omo-codex/plugin/components/rules/package.json +1 -1
  140. package/packages/omo-codex/plugin/components/rules/src/post-compact-budget.ts +1 -1
  141. package/packages/omo-codex/plugin/components/start-work-continuation/AGENTS.md +1 -0
  142. package/packages/omo-codex/plugin/components/start-work-continuation/README.md +5 -1
  143. package/packages/omo-codex/plugin/components/start-work-continuation/directive.md +2 -1
  144. package/packages/omo-codex/plugin/components/start-work-continuation/dist/cli.js +18 -0
  145. package/packages/omo-codex/plugin/components/start-work-continuation/hooks/hooks.json +2 -2
  146. package/packages/omo-codex/plugin/components/start-work-continuation/package.json +1 -1
  147. package/packages/omo-codex/plugin/components/start-work-continuation/src/codex-hook.ts +21 -0
  148. package/packages/omo-codex/plugin/components/start-work-continuation/test/codex-hook.test.ts +105 -0
  149. package/packages/omo-codex/plugin/components/teammode/hooks/hooks.json +1 -1
  150. package/packages/omo-codex/plugin/components/teammode/package.json +1 -1
  151. package/packages/omo-codex/plugin/components/telemetry/hooks/hooks.json +1 -1
  152. package/packages/omo-codex/plugin/components/telemetry/package.json +1 -1
  153. package/packages/omo-codex/plugin/components/ultrawork/CHANGELOG.md +2 -0
  154. package/packages/omo-codex/plugin/components/ultrawork/agents/lazycodex-code-reviewer.toml +1 -1
  155. package/packages/omo-codex/plugin/components/ultrawork/agents/lazycodex-gate-reviewer.toml +1 -1
  156. package/packages/omo-codex/plugin/components/ultrawork/agents/lazycodex-qa-executor.toml +1 -1
  157. package/packages/omo-codex/plugin/components/ultrawork/agents/lazycodex-worker-high.toml +1 -1
  158. package/packages/omo-codex/plugin/components/ultrawork/agents/lazycodex-worker-low.toml +1 -1
  159. package/packages/omo-codex/plugin/components/ultrawork/agents/lazycodex-worker-medium.toml +1 -1
  160. package/packages/omo-codex/plugin/components/ultrawork/agents/plan.toml +2 -2
  161. package/packages/omo-codex/plugin/components/ultrawork/directive.md +3 -2
  162. package/packages/omo-codex/plugin/components/ultrawork/hooks/hooks.json +1 -1
  163. package/packages/omo-codex/plugin/components/ultrawork/package.json +1 -1
  164. package/packages/omo-codex/plugin/components/ultrawork/skills/ultrawork/SKILL.md +3 -2
  165. package/packages/omo-codex/plugin/components/ultrawork/skills/ulw-plan/SKILL.md +2 -2
  166. package/packages/omo-codex/plugin/components/ultrawork/skills/ulw-plan/references/full-workflow.md +3 -3
  167. package/packages/omo-codex/plugin/components/ultrawork/skills/ulw-plan/scripts/scaffold-plan.mjs +1 -1
  168. package/packages/omo-codex/plugin/components/ulw-loop/AGENTS.md +1 -1
  169. package/packages/omo-codex/plugin/components/ulw-loop/CHANGELOG.md +2 -0
  170. package/packages/omo-codex/plugin/components/ulw-loop/README.md +11 -11
  171. package/packages/omo-codex/plugin/components/ulw-loop/directive.md +3 -2
  172. package/packages/omo-codex/plugin/components/ulw-loop/dist/checkpoint-reconciliation.js +1 -1
  173. package/packages/omo-codex/plugin/components/ulw-loop/dist/cli-output.d.ts +1 -1
  174. package/packages/omo-codex/plugin/components/ulw-loop/dist/cli-output.js +9 -9
  175. package/packages/omo-codex/plugin/components/ulw-loop/dist/cli-steering.js +1 -1
  176. package/packages/omo-codex/plugin/components/ulw-loop/dist/cli.js +66 -66
  177. package/packages/omo-codex/plugin/components/ulw-loop/dist/codex-goal-instruction.js +4 -4
  178. package/packages/omo-codex/plugin/components/ulw-loop/dist/codex-hook.js +1 -1
  179. package/packages/omo-codex/plugin/components/ulw-loop/dist/plan-crud.js +1 -1
  180. package/packages/omo-codex/plugin/components/ulw-loop/dist/plan-io.js +1 -1
  181. package/packages/omo-codex/plugin/components/ulw-loop/dist/steering.js +1 -1
  182. package/packages/omo-codex/plugin/components/ulw-loop/dist/stop-resume-hook.js +2 -2
  183. package/packages/omo-codex/plugin/components/ulw-loop/hooks/hooks.json +4 -4
  184. package/packages/omo-codex/plugin/components/ulw-loop/package.json +1 -1
  185. package/packages/omo-codex/plugin/components/ulw-loop/skills/ulw-loop/SKILL.md +3 -3
  186. package/packages/omo-codex/plugin/components/ulw-loop/skills/ulw-loop/references/full-workflow.md +22 -25
  187. package/packages/omo-codex/plugin/components/ulw-loop/src/checkpoint-reconciliation.ts +1 -1
  188. package/packages/omo-codex/plugin/components/ulw-loop/src/cli-output.ts +9 -9
  189. package/packages/omo-codex/plugin/components/ulw-loop/src/cli-steering.ts +1 -1
  190. package/packages/omo-codex/plugin/components/ulw-loop/src/cli.ts +1 -1
  191. package/packages/omo-codex/plugin/components/ulw-loop/src/codex-goal-instruction.ts +4 -4
  192. package/packages/omo-codex/plugin/components/ulw-loop/src/codex-hook.ts +1 -1
  193. package/packages/omo-codex/plugin/components/ulw-loop/src/plan-crud.ts +1 -1
  194. package/packages/omo-codex/plugin/components/ulw-loop/src/plan-io.ts +1 -1
  195. package/packages/omo-codex/plugin/components/ulw-loop/src/steering.ts +1 -1
  196. package/packages/omo-codex/plugin/components/ulw-loop/src/stop-resume-hook.ts +2 -2
  197. package/packages/omo-codex/plugin/components/ulw-loop/test/cli-commands.test.ts +2 -2
  198. package/packages/omo-codex/plugin/components/ulw-loop/test/cli-entrypoint.test.ts +1 -1
  199. package/packages/omo-codex/plugin/components/ulw-loop/test/cli-helpers.test.ts +2 -2
  200. package/packages/omo-codex/plugin/components/ulw-loop/test/cli-steering-kind-guidance.test.ts +1 -1
  201. package/packages/omo-codex/plugin/components/ulw-loop/test/codex-hook.test.ts +2 -2
  202. package/packages/omo-codex/plugin/components/ulw-loop/test/fixtures/quality-gate-builder.ts +1 -1
  203. package/packages/omo-codex/plugin/components/ulw-loop/test/package-smoke.test.ts +5 -5
  204. package/packages/omo-codex/plugin/components/ulw-loop/test/plan-io.test.ts +1 -1
  205. package/packages/omo-codex/plugin/components/ulw-loop/test/quality-gate-roles.test.ts +1 -1
  206. package/packages/omo-codex/plugin/components/ulw-loop/test/steering.test.ts +1 -1
  207. package/packages/omo-codex/plugin/components/ulw-loop/test/stop-resume-hook.test.ts +1 -1
  208. package/packages/omo-codex/plugin/hooks/post-compact-resetting-git-bash-mcp-reminder.json +1 -1
  209. package/packages/omo-codex/plugin/hooks/post-compact-resetting-lsp-diagnostics-cache.json +1 -1
  210. package/packages/omo-codex/plugin/hooks/post-compact-resetting-project-rule-cache.json +1 -1
  211. package/packages/omo-codex/plugin/hooks/post-tool-use-checking-codegraph-init-guidance.json +1 -1
  212. package/packages/omo-codex/plugin/hooks/post-tool-use-checking-comments.json +1 -1
  213. package/packages/omo-codex/plugin/hooks/post-tool-use-checking-lsp-diagnostics.json +1 -1
  214. package/packages/omo-codex/plugin/hooks/post-tool-use-checking-thread-title-hygiene.json +1 -1
  215. package/packages/omo-codex/plugin/hooks/post-tool-use-matching-project-rules.json +1 -1
  216. package/packages/omo-codex/plugin/hooks/pre-tool-use-enforcing-unlimited-goal-budget.json +1 -1
  217. package/packages/omo-codex/plugin/hooks/pre-tool-use-guarding-ulw-loop-spawns.json +1 -1
  218. package/packages/omo-codex/plugin/hooks/pre-tool-use-recommending-git-bash-mcp.json +1 -1
  219. package/packages/omo-codex/plugin/hooks/session-start-checking-auto-update.json +1 -1
  220. package/packages/omo-codex/plugin/hooks/session-start-checking-bootstrap-provisioning.json +1 -1
  221. package/packages/omo-codex/plugin/hooks/session-start-checking-codegraph-bootstrap.json +1 -1
  222. package/packages/omo-codex/plugin/hooks/session-start-loading-project-rules.json +1 -1
  223. package/packages/omo-codex/plugin/hooks/session-start-recording-session-telemetry.json +1 -1
  224. package/packages/omo-codex/plugin/hooks/stop-checking-start-work-continuation.json +1 -1
  225. package/packages/omo-codex/plugin/hooks/stop-checking-ulw-loop-resume.json +1 -1
  226. package/packages/omo-codex/plugin/hooks/subagent-stop-checking-start-work-continuation.json +1 -1
  227. package/packages/omo-codex/plugin/hooks/subagent-stop-verifying-lazycodex-executor-evidence.json +1 -1
  228. package/packages/omo-codex/plugin/hooks/user-prompt-submit-checking-ultrawork-trigger.json +1 -1
  229. package/packages/omo-codex/plugin/hooks/user-prompt-submit-checking-ulw-loop-steering.json +1 -1
  230. package/packages/omo-codex/plugin/hooks/user-prompt-submit-loading-project-rules.json +1 -1
  231. package/packages/omo-codex/plugin/package-lock.json +13 -13
  232. package/packages/omo-codex/plugin/package.json +1 -1
  233. package/packages/omo-codex/plugin/scripts/sync-skills.mjs +8 -2
  234. package/packages/omo-codex/plugin/skills/ast-grep/SOURCE +1 -1
  235. package/packages/omo-codex/plugin/skills/ast-grep/install.ps1 +2 -2
  236. package/packages/omo-codex/plugin/skills/ast-grep/install.sh +1 -1
  237. package/packages/omo-codex/plugin/skills/ast-grep/references/install.md +2 -2
  238. package/packages/omo-codex/plugin/skills/ast-grep/tests/smoke.sh +1 -1
  239. package/packages/omo-codex/plugin/skills/coding-agent-sessions/SKILL.md +4 -3
  240. package/packages/omo-codex/plugin/skills/coding-agent-sessions/references/all-platforms.md +3 -1
  241. package/packages/omo-codex/plugin/skills/coding-agent-sessions/scripts/agent_sessions/aside_scanner.py +140 -0
  242. package/packages/omo-codex/plugin/skills/coding-agent-sessions/scripts/agent_sessions/scanners.py +3 -0
  243. package/packages/omo-codex/plugin/skills/data-scientist/SKILL.md +243 -0
  244. package/packages/omo-codex/plugin/skills/data-scientist/agents/openai.yaml +2 -0
  245. package/packages/omo-codex/plugin/skills/data-scientist/references/common-scenarios.md +176 -0
  246. package/packages/omo-codex/plugin/skills/data-scientist/references/execution-templates.md +197 -0
  247. package/packages/omo-codex/plugin/skills/data-scientist/references/integration-patterns.md +153 -0
  248. package/packages/omo-codex/plugin/skills/data-scientist/references/performance-benchmarks.md +37 -0
  249. package/packages/omo-codex/plugin/skills/data-scientist/references/uv-setup.md +78 -0
  250. package/packages/omo-codex/plugin/skills/data-scientist/scripts/quick-query.py +111 -0
  251. package/packages/omo-codex/plugin/skills/data-scientist/scripts/setup-uv.ps1 +53 -0
  252. package/packages/omo-codex/plugin/skills/data-scientist/scripts/setup-uv.sh +60 -0
  253. package/packages/omo-codex/plugin/skills/debugging/SKILL.md +1 -1
  254. package/packages/omo-codex/plugin/skills/programming/SKILL.md +1 -2
  255. package/packages/omo-codex/plugin/skills/start-work/SKILL.md +54 -9
  256. package/packages/omo-codex/plugin/skills/ultimate-browsing/SKILL.md +2 -2
  257. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/__main__.py +8 -1
  258. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/bias_check.py +11 -0
  259. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/fetch_chain.py +90 -52
  260. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/result_schema.py +10 -1
  261. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/surrogate.py +214 -0
  262. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/surrogates.yaml +60 -0
  263. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/fixtures/amp_redirect_stub.html +7 -0
  264. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/fixtures/search_interstitial.html +19 -0
  265. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/fixtures/wayback_available.json +1 -0
  266. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/fixtures/wayback_snapshot.html +1128 -0
  267. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/test_surrogate.py +252 -0
  268. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/tests/test_surrogate_validators.py +78 -0
  269. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/validators.py +46 -0
  270. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/waf_detector.py +1 -1
  271. package/packages/omo-codex/plugin/skills/ultimate-browsing/engine/waf_profiles.yaml +10 -5
  272. package/packages/omo-codex/plugin/skills/ultimate-browsing/references/agent-reach/social.md +1 -1
  273. package/packages/omo-codex/plugin/skills/ultimate-browsing/references/chrome-stealth.md +13 -11
  274. package/packages/omo-codex/plugin/skills/ultimate-browsing/references/insane-search/README.md +4 -4
  275. package/packages/omo-codex/plugin/skills/ultimate-browsing/references/insane-search/cache-archive.md +51 -50
  276. package/packages/omo-codex/plugin/skills/ultimate-browsing/references/insane-search/fallback.md +1 -1
  277. package/packages/omo-codex/plugin/skills/ultimate-browsing/references/insane-search/jina.md +8 -2
  278. package/packages/omo-codex/plugin/skills/ultimate-browsing/references/insane-search/naver.md +1 -1
  279. package/packages/omo-codex/plugin/skills/ultimate-browsing/references/insane-search/twitter.md +3 -3
  280. package/packages/omo-codex/plugin/skills/ultrawork/SKILL.md +3 -2
  281. package/packages/omo-codex/plugin/skills/ulw-loop/SKILL.md +3 -3
  282. package/packages/omo-codex/plugin/skills/ulw-loop/references/full-workflow.md +22 -25
  283. package/packages/omo-codex/plugin/skills/ulw-plan/SKILL.md +2 -2
  284. package/packages/omo-codex/plugin/skills/ulw-plan/references/full-workflow.md +3 -3
  285. package/packages/omo-codex/plugin/skills/ulw-plan/scripts/scaffold-plan.mjs +1 -1
  286. package/packages/omo-codex/plugin/skills/ulw-research/SKILL.md +127 -12
  287. package/packages/omo-codex/plugin/test/bootstrap-binlinks.test.mjs +12 -12
  288. package/packages/omo-codex/plugin/test/bootstrap-orchestration.test.mjs +36 -4
  289. package/packages/omo-codex/plugin/test/sync-skills-orchestration.test.mjs +1 -1
  290. package/packages/omo-codex/plugin/test/sync-skills-test-support.mjs +9 -2
  291. package/packages/omo-codex/plugin/test/sync-skills.test.mjs +12 -0
  292. package/packages/omo-codex/scripts/install-bin-links.test.mjs +56 -2
  293. package/packages/omo-codex/scripts/install-delegated-command.test.mjs +6 -6
  294. package/packages/omo-codex/scripts/install-dist/install-local.mjs +138 -65
  295. package/packages/omo-codex/scripts/install-local-entrypoint.test.mjs +4 -4
  296. package/packages/omo-codex/scripts/install-local.test.mjs +5 -2
  297. package/packages/shared-skills/index.mjs +19 -1
  298. package/packages/shared-skills/skills/ast-grep/SOURCE +1 -1
  299. package/packages/shared-skills/skills/ast-grep/install.ps1 +2 -2
  300. package/packages/shared-skills/skills/ast-grep/install.sh +1 -1
  301. package/packages/shared-skills/skills/ast-grep/references/install.md +2 -2
  302. package/packages/shared-skills/skills/ast-grep/tests/smoke.sh +1 -1
  303. package/packages/shared-skills/skills/coding-agent-sessions/SKILL.md +4 -3
  304. package/packages/shared-skills/skills/coding-agent-sessions/references/all-platforms.md +3 -1
  305. package/packages/shared-skills/skills/coding-agent-sessions/scripts/agent_sessions/aside_scanner.py +140 -0
  306. package/packages/shared-skills/skills/coding-agent-sessions/scripts/agent_sessions/scanners.py +3 -0
  307. package/packages/shared-skills/skills/data-scientist/SKILL.md +243 -0
  308. package/packages/shared-skills/skills/data-scientist/references/common-scenarios.md +176 -0
  309. package/packages/shared-skills/skills/data-scientist/references/execution-templates.md +197 -0
  310. package/packages/shared-skills/skills/data-scientist/references/integration-patterns.md +153 -0
  311. package/packages/shared-skills/skills/data-scientist/references/performance-benchmarks.md +37 -0
  312. package/packages/shared-skills/skills/data-scientist/references/uv-setup.md +78 -0
  313. package/packages/shared-skills/skills/data-scientist/scripts/quick-query.py +111 -0
  314. package/packages/shared-skills/skills/data-scientist/scripts/setup-uv.ps1 +53 -0
  315. package/packages/shared-skills/skills/data-scientist/scripts/setup-uv.sh +60 -0
  316. package/packages/shared-skills/skills/debugging/SKILL.md +1 -1
  317. package/packages/shared-skills/skills/programming/SKILL.md +1 -2
  318. package/packages/shared-skills/skills/start-work/SKILL.md +54 -9
  319. package/packages/shared-skills/skills/ultimate-browsing/SKILL.md +2 -2
  320. package/packages/shared-skills/skills/ultimate-browsing/engine/__main__.py +8 -1
  321. package/packages/shared-skills/skills/ultimate-browsing/engine/bias_check.py +11 -0
  322. package/packages/shared-skills/skills/ultimate-browsing/engine/fetch_chain.py +90 -52
  323. package/packages/shared-skills/skills/ultimate-browsing/engine/result_schema.py +10 -1
  324. package/packages/shared-skills/skills/ultimate-browsing/engine/surrogate.py +214 -0
  325. package/packages/shared-skills/skills/ultimate-browsing/engine/surrogates.yaml +60 -0
  326. package/packages/shared-skills/skills/ultimate-browsing/engine/tests/fixtures/amp_redirect_stub.html +7 -0
  327. package/packages/shared-skills/skills/ultimate-browsing/engine/tests/fixtures/search_interstitial.html +19 -0
  328. package/packages/shared-skills/skills/ultimate-browsing/engine/tests/fixtures/wayback_available.json +1 -0
  329. package/packages/shared-skills/skills/ultimate-browsing/engine/tests/fixtures/wayback_snapshot.html +1128 -0
  330. package/packages/shared-skills/skills/ultimate-browsing/engine/tests/test_surrogate.py +252 -0
  331. package/packages/shared-skills/skills/ultimate-browsing/engine/tests/test_surrogate_validators.py +78 -0
  332. package/packages/shared-skills/skills/ultimate-browsing/engine/validators.py +46 -0
  333. package/packages/shared-skills/skills/ultimate-browsing/engine/waf_detector.py +1 -1
  334. package/packages/shared-skills/skills/ultimate-browsing/engine/waf_profiles.yaml +10 -5
  335. package/packages/shared-skills/skills/ultimate-browsing/references/agent-reach/social.md +1 -1
  336. package/packages/shared-skills/skills/ultimate-browsing/references/chrome-stealth.md +13 -11
  337. package/packages/shared-skills/skills/ultimate-browsing/references/insane-search/README.md +4 -4
  338. package/packages/shared-skills/skills/ultimate-browsing/references/insane-search/cache-archive.md +51 -50
  339. package/packages/shared-skills/skills/ultimate-browsing/references/insane-search/fallback.md +1 -1
  340. package/packages/shared-skills/skills/ultimate-browsing/references/insane-search/jina.md +8 -2
  341. package/packages/shared-skills/skills/ultimate-browsing/references/insane-search/naver.md +1 -1
  342. package/packages/shared-skills/skills/ultimate-browsing/references/insane-search/twitter.md +3 -3
  343. package/packages/shared-skills/skills/ulw-plan/SKILL.md +2 -2
  344. package/packages/shared-skills/skills/ulw-plan/references/full-workflow.md +3 -3
  345. package/packages/shared-skills/skills/ulw-plan/scripts/scaffold-plan.mjs +1 -1
  346. package/packages/shared-skills/skills/ulw-research/SKILL.md +128 -12
  347. package/postinstall.mjs +6 -0
@@ -37,7 +37,7 @@ Read the matching reference before acting: [`references/insane-search/README.md`
37
37
  ## Tier 1 — insane-search (headless extraction)
38
38
 
39
39
  **When**: content extraction, blocked-URL bypass, media metadata — no browser UI needed.
40
- **Why first**: ~10x faster than a browser, no process spin-up; handles most "fetch this blocked page" requests via curl_cffi TLS impersonation, yt-dlp (1858 sites), Jina Reader, official public APIs, mobile URL transforms, and a Playwright real-Chrome fallback. The engine lives **inside this skill** at `engine/` and is invoked as a module.
40
+ **Why first**: ~10x faster than a browser, no process spin-up; handles most "fetch this blocked page" requests via curl_cffi TLS impersonation, yt-dlp (1858 sites), official public APIs, mobile URL transforms, **Phase-2.5 surrogate archives** (Wayback / archive.today snapshots, provenance-tagged — see [`references/insane-search/cache-archive.md`](references/insane-search/cache-archive.md)), a key-gated Jina Reader (`JINA_API_KEY`), and a Playwright real-Chrome fallback. The engine lives **inside this skill** at `engine/` and is invoked as a module. Surrogate results are dated COPIES: a result whose `provenance` is `snapshot` must be reported with its `snapshot_timestamp`, never presented as the live page.
41
41
 
42
42
  ```bash
43
43
  # Core command — auto-detects WAF, runs the full fetch grid (run from the skill dir):
@@ -60,7 +60,7 @@ The full engine harness (rules R1-R7, the Phase 0 official-API index, the no-sit
60
60
 
61
61
  ## Tier 1.5 — agent-reach (platform-native readers)
62
62
 
63
- **When**: the target is a platform with a first-class API/CLI that beats generic fetching — especially Chinese platforms that stealth browsers still cannot reach cleanly. Several channels are zero-config (Douyin, Weibo via Jina, V2EX, Reddit, Jina Reader, RSS, YouTube); others need a one-time auth you supply via environment variables if you have access.
63
+ **When**: the target is a platform with a first-class API/CLI that beats generic fetching — especially Chinese platforms that stealth browsers still cannot reach cleanly. Several channels are zero-config (Douyin, V2EX, Reddit, RSS, YouTube); others need a one-time auth you supply via environment variables if you have access (`JINA_API_KEY` for Jina Reader — anonymous access is dead, see `references/insane-search/jina.md`; `TWITTER_*` for X; a transcription key for podcasts).
64
64
 
65
65
  | Category | Platforms | Entry |
66
66
  |---|---|---|
@@ -39,6 +39,10 @@ def build_parser() -> argparse.ArgumentParser:
39
39
  help="Upper bound across all phases (default 12).")
40
40
  p.add_argument("--no-playwright", action="store_true",
41
41
  help="Skip Playwright fallback (curl-only).")
42
+ p.add_argument("--allow-proxy", action="store_true",
43
+ help=("Enable kind=proxy surrogate entries (raw relay routes; "
44
+ "MITM by construction — never citable alone, never "
45
+ "receives cookies/auth headers). Off by default."))
42
46
  p.add_argument("--json", action="store_true",
43
47
  help="Emit FetchResult as JSON to stdout (content omitted).")
44
48
  p.add_argument("--trace", action="store_true",
@@ -56,6 +60,7 @@ def main(argv: list[str] | None = None) -> int:
56
60
  timeout=args.timeout,
57
61
  max_attempts=args.max_attempts,
58
62
  enable_playwright=not args.no_playwright,
63
+ allow_surrogate_proxy=args.allow_proxy,
59
64
  )
60
65
  except Exception as e:
61
66
  print(f"engine fatal: {type(e).__name__}: {e}", file=sys.stderr)
@@ -96,7 +101,9 @@ def main(argv: list[str] | None = None) -> int:
96
101
  # Default: HTML to stdout, status to stderr.
97
102
  print(result.content, end="")
98
103
  print(f"\n[engine] ok={result.ok} verdict={result.verdict} "
99
- f"profile={result.profile_used} attempts={len(result.trace)}",
104
+ f"profile={result.profile_used} attempts={len(result.trace)} "
105
+ f"provenance={result.provenance}"
106
+ + (f" snapshot_ts={result.snapshot_timestamp}" if result.snapshot_timestamp else ""),
100
107
  file=sys.stderr)
101
108
 
102
109
  return 0 if result.ok else 1
@@ -54,6 +54,14 @@ URL_ALLOWLIST = {
54
54
  "www.google.com", "google.com",
55
55
  # Generic HTTP test endpoint for infrastructure / transport tests.
56
56
  "httpbin.org",
57
+ # Surrogate-tier infrastructure (engine/surrogates.yaml): archives/relays
58
+ # that serve a copy of ANY target URL. Same no-site-preference category as
59
+ # the google.com Referer — they are not target sites.
60
+ "archive.org", "web.archive.org",
61
+ "r.jina.ai",
62
+ # Placeholder host in the proxy-entry shape example; "canonical non-routable
63
+ # doc domain" (.invalid TLD), can never be a real target.
64
+ "relay.invalid",
57
65
  }
58
66
 
59
67
  # Files / dirs that must be clean.
@@ -80,6 +88,9 @@ EXPLICIT_ALLOW_FILES = {
80
88
  # None right now — add if needed with justification.
81
89
  }
82
90
 
91
+ # NOTE-BIAS-OK convention for surrogate tests: fixture URLs (stub/interstitial
92
+ # captures from third-party services) carry the marker on the flagged line.
93
+
83
94
 
84
95
  def _line_is_exempt(line: str, ext: str) -> bool:
85
96
  markers = COMMENT_OK_MARKERS.get(ext, ())
@@ -40,6 +40,7 @@ def fetch(
40
40
  timeout: int = 25,
41
41
  max_attempts: int = 12,
42
42
  enable_playwright: bool = True, # hook left for executor module
43
+ allow_surrogate_proxy: bool = False, # opt-in: kind=proxy registry entries
43
44
  ) -> FetchResult:
44
45
  """Fetch `url` using the generic grid.
45
46
 
@@ -175,73 +176,110 @@ def fetch(
175
176
  if att.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
176
177
  return _build_result(resp, att, trace, profile_used=profile_id)
177
178
 
178
- # -------- Phase 3: Playwright fallback (profile-driven order) -----------
179
- if enable_playwright:
180
- try:
181
- from .executor import run_playwright_fallback # lazy import
182
- # Honour profile's `fallback_when_challenge` list — iterate the
183
- # caller-declared order instead of capability-inferred single pick.
184
- fb_profile = load_profile(profile_used or "unknown_challenge", profiles=profiles)
185
- fb_order = fb_profile.get("fallback_when_challenge") or ["playwright_real_chrome"]
186
- pw_attempt = None
187
- pw_content = ""
188
- for fb_name in fb_order:
189
- if fb_name == "curl_grid_exhaust":
190
- # Already performed in Phase 2; nothing more to do here.
191
- continue
192
- pw_attempt, pw_content = run_playwright_fallback(
179
+ # -------- Phase 2.5 + 3: fallback ladder (profile-driven order) ---------
180
+ # Surrogate routes are NOT browser work, so they run even when the caller
181
+ # disabled Playwright; only browser executors honour `enable_playwright`.
182
+ try:
183
+ # Honour profile's `fallback_when_challenge` list — iterate the
184
+ # caller-declared order instead of capability-inferred single pick.
185
+ fb_profile = load_profile(profile_used or "unknown_challenge", profiles=profiles)
186
+ fb_order = fb_profile.get("fallback_when_challenge") or ["playwright_real_chrome"]
187
+ pw_attempt = None
188
+ pw_content = ""
189
+ for fb_name in fb_order:
190
+ if fb_name == "curl_grid_exhaust":
191
+ # Already performed in Phase 2; nothing more to do here.
192
+ continue
193
+ if fb_name.startswith("surrogate_"):
194
+ # Phase 2.5: archive/reader/proxy routes serving a copy of
195
+ # the target. Cheaper than a browser; provenance-labeled.
196
+ from . import surrogate as surrogate_mod
197
+ s_atts, s_meta, s_content = surrogate_mod.run_surrogate(
193
198
  url,
194
- profile_id=profile_used or "unknown_challenge",
199
+ registry=surrogate_mod.load_surrogates(),
200
+ allow_proxy=allow_surrogate_proxy,
201
+ timeout=timeout,
195
202
  success_selectors=success_selectors,
196
- device_class=device_class,
197
- force_executor=fb_name,
198
203
  )
199
- trace.append(pw_attempt)
200
- if pw_attempt.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
204
+ trace.extend(s_atts)
205
+ s_att = s_atts[-1]
206
+ if s_att.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
201
207
  return FetchResult(
202
208
  ok=True,
203
- content=pw_content,
204
- final_url=pw_attempt.url,
205
- verdict=pw_attempt.verdict,
209
+ content=s_content,
210
+ final_url=s_att.url,
211
+ verdict=s_att.verdict,
206
212
  profile_used=profile_used,
207
213
  trace=trace,
208
- summary=f"Playwright fallback succeeded via {fb_name}",
214
+ summary=(
215
+ f"surrogate {s_meta.get('surrogate')} succeeded"
216
+ f" (provenance={s_meta.get('provenance')}"
217
+ + (f" snapshot_ts={s_meta.get('snapshot_timestamp')})" if s_meta.get('snapshot_timestamp') else ")")
218
+ ),
219
+ provenance=str(s_meta.get("provenance", "live")),
220
+ snapshot_timestamp=s_meta.get("snapshot_timestamp"),
221
+ trust=str(s_meta.get("trust", "origin")),
209
222
  )
210
- # Synthesize a placeholder if no iteration ran (empty list).
211
- if pw_attempt is None:
212
- pw_attempt = Attempt(
213
- phase="fallback",
214
- executor="none",
215
- url=url,
216
- url_transform="original",
217
- impersonate=None,
218
- referer="",
219
- verdict=Verdict.UNKNOWN.value,
220
- error="profile has empty fallback_when_challenge",
223
+ continue
224
+ if not enable_playwright:
225
+ continue
226
+ from .executor import run_playwright_fallback # lazy import
227
+ pw_attempt, pw_content = run_playwright_fallback(
228
+ url,
229
+ profile_id=profile_used or "unknown_challenge",
230
+ success_selectors=success_selectors,
231
+ device_class=device_class,
232
+ force_executor=fb_name,
233
+ )
234
+ trace.append(pw_attempt)
235
+ if pw_attempt.verdict in (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value):
236
+ return FetchResult(
237
+ ok=True,
238
+ content=pw_content,
239
+ final_url=pw_attempt.url,
240
+ verdict=pw_attempt.verdict,
241
+ profile_used=profile_used,
242
+ trace=trace,
243
+ summary=f"Playwright fallback succeeded via {fb_name}",
221
244
  )
222
- trace.append(pw_attempt)
223
- except ImportError:
224
- trace.append(Attempt(
245
+ # Synthesize a placeholder only when the profile genuinely offers no
246
+ # fallback route. Entries skipped because the caller disabled the
247
+ # browser are a caller choice, not a profile defect — no trace noise.
248
+ actionable = [n for n in fb_order if n != "curl_grid_exhaust"]
249
+ if pw_attempt is None and not actionable:
250
+ pw_attempt = Attempt(
225
251
  phase="fallback",
226
- executor="playwright",
252
+ executor="none",
227
253
  url=url,
228
254
  url_transform="original",
229
255
  impersonate=None,
230
256
  referer="",
231
257
  verdict=Verdict.UNKNOWN.value,
232
- error="executor module not available",
233
- ))
234
- except (RuntimeError, OSError) as e:
235
- trace.append(Attempt(
236
- phase="fallback",
237
- executor="playwright",
238
- url=url,
239
- url_transform="original",
240
- impersonate=None,
241
- referer="",
242
- verdict=Verdict.UNKNOWN.value,
243
- error=f"{type(e).__name__}:{str(e)[:200]}",
244
- ))
258
+ error="profile has empty fallback_when_challenge",
259
+ )
260
+ trace.append(pw_attempt)
261
+ except ImportError:
262
+ trace.append(Attempt(
263
+ phase="fallback",
264
+ executor="playwright",
265
+ url=url,
266
+ url_transform="original",
267
+ impersonate=None,
268
+ referer="",
269
+ verdict=Verdict.UNKNOWN.value,
270
+ error="executor module not available",
271
+ ))
272
+ except (RuntimeError, OSError) as e:
273
+ trace.append(Attempt(
274
+ phase="fallback",
275
+ executor="playwright",
276
+ url=url,
277
+ url_transform="original",
278
+ impersonate=None,
279
+ referer="",
280
+ verdict=Verdict.UNKNOWN.value,
281
+ error=f"{type(e).__name__}:{str(e)[:200]}",
282
+ ))
245
283
 
246
284
  # -------- Give up, return best we have ----------------------------------
247
285
  summary = format_summary(trace, profile_used)
@@ -1,7 +1,10 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  from dataclasses import asdict, dataclass, field
4
- from typing import Optional
4
+ from typing import Literal, Optional
5
+
6
+ Provenance = Literal["live", "snapshot", "proxy"]
7
+ Trust = Literal["origin", "archive", "untrusted"]
5
8
 
6
9
 
7
10
  @dataclass
@@ -32,6 +35,9 @@ class FetchResult:
32
35
  profile_used: Optional[str] = None
33
36
  trace: list[Attempt] = field(default_factory=list)
34
37
  summary: str = ""
38
+ provenance: str = "live" # live | snapshot | proxy
39
+ snapshot_timestamp: Optional[str] = None # archive's own timestamp, when snapshot
40
+ trust: str = "origin" # origin | archive | untrusted
35
41
 
36
42
  def to_dict(self) -> dict:
37
43
  return {
@@ -41,5 +47,8 @@ class FetchResult:
41
47
  "profile_used": self.profile_used,
42
48
  "trace": [a.to_dict() for a in self.trace],
43
49
  "summary": self.summary,
50
+ "provenance": self.provenance,
51
+ "snapshot_timestamp": self.snapshot_timestamp,
52
+ "trust": self.trust,
44
53
  "content_length": len(self.content),
45
54
  }
@@ -0,0 +1,214 @@
1
+ """Phase 2.5 surrogate retrieval: archive / reader / proxy routes for blocked origins.
2
+
3
+ When the curl grid cannot fetch the live page, this module reads a declarative
4
+ registry (engine/surrogates.yaml) and tries third-party routes that serve a
5
+ copy of the target instead. Provenance semantics are contractual:
6
+
7
+ kind=archive -> provenance="snapshot" (+ archive's own timestamp), trust=archive
8
+ kind=reader -> provenance="live" (server-side re-render of the live page)
9
+ kind=proxy -> provenance="proxy", trust=untrusted; requires allow_proxy=True
10
+ and never sends Cookie/Authorization headers (relay = MITM)
11
+
12
+ Every response body passes through validators.validate() with target_url set,
13
+ so AMP-style redirect stubs and search-engine interstitials are rejected
14
+ rather than recorded as wins. Entries whose `last_verified` is older than
15
+ MAX_STALE_DAYS are tried after fresh ones (routes rot faster than they are
16
+ maintained).
17
+ """
18
+ from __future__ import annotations
19
+
20
+ import json
21
+ import os
22
+ import time
23
+ from datetime import date
24
+
25
+ try:
26
+ import yaml
27
+ except ImportError:
28
+ yaml = None
29
+
30
+ from .result_schema import Attempt
31
+ from .validators import Verdict, validate
32
+
33
+ SURROGATES_PATH = os.path.join(os.path.dirname(__file__), "surrogates.yaml")
34
+ MAX_STALE_DAYS = 90
35
+
36
+ DEFAULT_SURROGATES: dict = {
37
+ "wayback": {
38
+ "kind": "archive",
39
+ "trust": "archive",
40
+ "enabled": True,
41
+ "last_verified": "2026-08-09",
42
+ "discovery": {
43
+ "url": "https://archive.org/wayback/available?url={target}",
44
+ "flag_pointer": "archived_snapshots.closest.available",
45
+ "snapshot_url_pointer": "archived_snapshots.closest.url",
46
+ "snapshot_timestamp_pointer": "archived_snapshots.closest.timestamp",
47
+ },
48
+ "min_body_bytes": 3072,
49
+ },
50
+ }
51
+
52
+
53
+ def load_surrogates(path: str = SURROGATES_PATH) -> dict:
54
+ if yaml is None:
55
+ return dict(DEFAULT_SURROGATES)
56
+ try:
57
+ with open(path, encoding="utf-8") as fh:
58
+ loaded = yaml.safe_load(fh) or {}
59
+ except (OSError, yaml.YAMLError):
60
+ return dict(DEFAULT_SURROGATES)
61
+ if not isinstance(loaded, dict):
62
+ return dict(DEFAULT_SURROGATES)
63
+ usable = {k: v for k, v in loaded.items() if isinstance(v, dict) and k}
64
+ return usable or dict(DEFAULT_SURROGATES)
65
+
66
+
67
+ def is_stale(entry: dict, max_age_days: int = MAX_STALE_DAYS, today: date | None = None) -> bool:
68
+ verified = entry.get("last_verified")
69
+ if isinstance(verified, date):
70
+ parsed = verified
71
+ elif isinstance(verified, str):
72
+ try:
73
+ parsed = date.fromisoformat(verified)
74
+ except ValueError:
75
+ return True
76
+ else:
77
+ return True
78
+ return ((today or date.today()) - parsed).days > max_age_days
79
+
80
+
81
+ def _http_get(url: str, *, headers: dict | None = None, timeout: int = 25):
82
+ try:
83
+ from curl_cffi import requests as cffi_requests
84
+ except ImportError as exc:
85
+ raise RuntimeError("curl_cffi not installed") from exc
86
+ return cffi_requests.get(
87
+ url,
88
+ impersonate="chrome",
89
+ headers=headers or {"Accept": "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8"},
90
+ timeout=timeout,
91
+ allow_redirects=True,
92
+ )
93
+
94
+
95
+ def _json_pointer(obj, dotted: str):
96
+ current = obj
97
+ for part in dotted.split("."):
98
+ if not isinstance(current, dict):
99
+ return None
100
+ current = current.get(part)
101
+ return current
102
+
103
+
104
+ def _provenance_of(kind: str) -> str:
105
+ if kind == "archive":
106
+ return "snapshot"
107
+ if kind == "proxy":
108
+ return "proxy"
109
+ return "live"
110
+
111
+
112
+ def _skip_reason(name: str, entry: dict, allow_proxy: bool) -> str | None:
113
+ kind = entry.get("kind", "archive")
114
+ if kind == "proxy" and not allow_proxy:
115
+ return "proxy_requires_allow_proxy_flag"
116
+ required_env = entry.get("enabled_env")
117
+ if required_env and not os.environ.get(str(required_env)):
118
+ return f"missing_env:{required_env}"
119
+ if entry.get("enabled") is False and kind != "proxy":
120
+ return "disabled"
121
+ return None
122
+
123
+
124
+ def _attempt(name: str, url: str, verdict: Verdict, error=None, status: int = 0, size: int = 0, elapsed: float = 0.0) -> Attempt:
125
+ return Attempt(
126
+ phase="surrogate",
127
+ executor=f"surrogate_{name}",
128
+ url=url,
129
+ url_transform="original",
130
+ impersonate=None,
131
+ referer="",
132
+ status=status,
133
+ body_size=size,
134
+ verdict=verdict.value,
135
+ error=(str(error)[:200] if error else None),
136
+ elapsed_s=round(elapsed, 3),
137
+ )
138
+
139
+
140
+ def _resolve_target(name: str, entry: dict, target: str, timeout: int) -> tuple[str | None, str | None]:
141
+ discovery = entry.get("discovery")
142
+ if not discovery:
143
+ host_rotation = entry.get("host_rotation") or [None]
144
+ host = host_rotation[0]
145
+ template = entry.get("fetch")
146
+ if not template or host is None and "{host}" in template:
147
+ return None, None
148
+ return template.format(host=host, target=target), None
149
+ discovery_url = discovery.get("url", "").format(target=target)
150
+ resp = _http_get(discovery_url, timeout=timeout)
151
+ payload = resp.json()
152
+ if discovery.get("flag_pointer") and not _json_pointer(payload, discovery["flag_pointer"]):
153
+ return None, None
154
+ snap_url = _json_pointer(payload, discovery.get("snapshot_url_pointer", ""))
155
+ timestamp = _json_pointer(payload, discovery.get("snapshot_timestamp_pointer", "")) or None
156
+ return snap_url, timestamp
157
+
158
+
159
+ def run_surrogate(
160
+ target: str,
161
+ *,
162
+ registry: dict | None = None,
163
+ allow_proxy: bool = False,
164
+ timeout: int = 25,
165
+ success_selectors: list[str] | None = None,
166
+ ) -> tuple[list[Attempt], dict, str]:
167
+ registry = dict(registry) if registry else load_surrogates()
168
+ ordered = sorted(registry.items(), key=lambda kv: int(is_stale(kv[1])))
169
+ live_meta = {"provenance": "live", "snapshot_timestamp": None, "trust": "origin", "surrogate": None}
170
+ attempts: list[Attempt] = []
171
+
172
+ for name, entry in ordered:
173
+ kind = entry.get("kind", "archive")
174
+ reason = _skip_reason(name, entry, allow_proxy)
175
+ if reason:
176
+ attempts.append(_attempt(name, target, Verdict.UNKNOWN, error=f"skipped:{reason}"))
177
+ continue
178
+ started = time.time()
179
+ try:
180
+ fetch_url, timestamp = _resolve_target(name, entry, target, timeout)
181
+ except Exception as exc:
182
+ attempts.append(_attempt(name, target, Verdict.UNKNOWN, error=f"{type(exc).__name__}:{exc}", elapsed=time.time() - started))
183
+ continue
184
+ if not fetch_url:
185
+ attempts.append(_attempt(name, target, Verdict.UNKNOWN, error="no snapshot available", elapsed=time.time() - started))
186
+ continue
187
+ try:
188
+ resp = _http_get(fetch_url, timeout=timeout)
189
+ except Exception as exc:
190
+ attempts.append(_attempt(name, fetch_url, Verdict.UNKNOWN, error=f"{type(exc).__name__}:{exc}", elapsed=time.time() - started))
191
+ continue
192
+ body = getattr(resp, "text", "") or ""
193
+ status = int(getattr(resp, "status_code", 0) or 0)
194
+ min_bytes = int(entry.get("min_body_bytes", 3072))
195
+ att = _attempt(name, fetch_url, Verdict.UNKNOWN, status=status, size=len(body), elapsed=time.time() - started)
196
+ attempts.append(att)
197
+ if status >= 400 or len(body) < min_bytes:
198
+ att.error = f"surrogate_unusable:status={status},size={len(body)}"
199
+ continue
200
+ vr = validate(resp, success_selectors=success_selectors, target_url=target)
201
+ att.verdict = vr.verdict.value
202
+ att.reasons = vr.reasons
203
+ if vr.ok:
204
+ meta = {
205
+ "provenance": _provenance_of(kind),
206
+ "snapshot_timestamp": timestamp if kind == "archive" else None,
207
+ "trust": str(entry.get("trust", "archive")),
208
+ "surrogate": name,
209
+ }
210
+ return attempts, meta, body
211
+
212
+ if not attempts:
213
+ attempts.append(_attempt("none", target, Verdict.UNKNOWN, error="no surrogate entries"))
214
+ return attempts, live_meta, ""
@@ -0,0 +1,60 @@
1
+ # Surrogate routes — third-party services serving a COPY of a page when the
2
+ # live origin blocks the fetch chain (Phase 2.5, between grid and Playwright).
3
+ #
4
+ # Rules (same discipline as waf_profiles.yaml):
5
+ # * Site-agnostic infrastructure ONLY (No-Site-Name Rule): every host here
6
+ # must work for any target URL. No target-site domains, selectors, brand
7
+ # names.
8
+ # * Surrogate routes ROT — the 2026-08 probe found 4 of 6 known routes dead
9
+ # or returning stubs. Entries must carry `last_verified` (ISO date) and a
10
+ # passing liveness record in the PR that adds them; refresh quarterly.
11
+ # Entries older than 90 days are tried after fresh ones and flagged.
12
+ # * kind semantics:
13
+ # archive — serves a dated snapshot; provenance=snapshot, trust=archive.
14
+ # reader — re-renders the LIVE page server-side; provenance=live.
15
+ # proxy — raw forwarding relay; MITM by construction, trust=untrusted,
16
+ # skipped unless the caller passes --allow-proxy, and never
17
+ # receives Cookie/Authorization headers.
18
+ # * keys:
19
+ # discovery.json_pointer — dotted path into the discovery JSON payload.
20
+ # host_rotation — try hostile-domain cousins in order (one may be
21
+ # blocked while another answers).
22
+ # enabled_env — entry is inert until that env var is set (key-gated).
23
+ # min_body_bytes — response floor; below it the body is a stub/error.
24
+ wayback:
25
+ kind: archive
26
+ trust: archive
27
+ enabled: true
28
+ last_verified: 2026-08-09
29
+ discovery:
30
+ url: "https://archive.org/wayback/available?url={target}"
31
+ flag_pointer: "archived_snapshots.closest.available"
32
+ snapshot_url_pointer: "archived_snapshots.closest.url"
33
+ snapshot_timestamp_pointer: "archived_snapshots.closest.timestamp"
34
+ min_body_bytes: 3072
35
+
36
+ archive_today:
37
+ kind: archive
38
+ trust: archive
39
+ enabled: true
40
+ last_verified: 2026-08-09
41
+ host_rotation: ["archive.ph", "archive.md", "archive.li", "archive.is"]
42
+ fetch: "https://{host}/newest/{target}"
43
+ min_body_bytes: 3072
44
+
45
+ jina_reader:
46
+ kind: reader
47
+ trust: archive
48
+ enabled: true
49
+ enabled_env: JINA_API_KEY
50
+ last_verified: 2026-08-09
51
+ fetch: "https://r.jina.ai/{target}"
52
+ min_body_bytes: 512
53
+
54
+ anon_relay_example:
55
+ kind: proxy
56
+ trust: untrusted
57
+ enabled: false
58
+ last_verified: 2026-08-09
59
+ fetch: "https://relay.invalid/?u={target}"
60
+ min_body_bytes: 3072
@@ -0,0 +1,7 @@
1
+ <HTML><HEAD>
2
+ <meta http-equiv="content-type" content="text/html;charset=utf-8">
3
+ <TITLE>Redirecting</TITLE>
4
+ <META HTTP-EQUIV="refresh" content="0; url=https://en.wikipedia.org/wiki/Web_archiving">
5
+ </HEAD>
6
+ <BODY onLoad="location.replace('https://en.wikipedia.org/wiki/Web_archiving'+document.location.hash)">
7
+ </BODY></HTML>