opendevbrowser 0.0.15 → 0.0.16

package/skills/opendevbrowser-best-practices/scripts/run-robustness-audit.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+skills_root="$(cd "$root/.." && pwd)"
+matrix_ref="../opendevbrowser-best-practices/artifacts/browser-agent-known-issues-matrix.md"
+
+usage() {
+  cat <<'HELP'
+Run robustness audit checks across opendevbrowser workflow skill packs.
+
+Usage:
+  ./skills/opendevbrowser-best-practices/scripts/run-robustness-audit.sh [skill-name]
+
+Examples:
+  ./skills/opendevbrowser-best-practices/scripts/run-robustness-audit.sh
+  ./skills/opendevbrowser-best-practices/scripts/run-robustness-audit.sh opendevbrowser-shopping
+HELP
+}
+
+skills=(
+  "opendevbrowser-login-automation:ISSUE-01 ISSUE-02 ISSUE-03 ISSUE-04 ISSUE-05 ISSUE-06 ISSUE-07"
+  "opendevbrowser-form-testing:ISSUE-01 ISSUE-02 ISSUE-05 ISSUE-06 ISSUE-08"
+  "opendevbrowser-data-extraction:ISSUE-01 ISSUE-06 ISSUE-08 ISSUE-09 ISSUE-10"
+  "opendevbrowser-shopping:ISSUE-06 ISSUE-09 ISSUE-10 ISSUE-11 ISSUE-12"
+  "opendevbrowser-product-presentation-asset:ISSUE-10 ISSUE-11 ISSUE-12"
+  "opendevbrowser-research:ISSUE-06 ISSUE-09 ISSUE-10 ISSUE-12"
+)
+
+status=0
+target="${1:-all}"
+
+if [[ "$target" == "-h" || "$target" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+found_target=0
+for entry in "${skills[@]}"; do
+  skill="${entry%%:*}"
+  expected="${entry#*:}"
+  if [[ "$target" != "all" && "$skill" != "$target" ]]; then
+    continue
+  fi
+  found_target=1
+
+  skill_file="$skills_root/$skill/SKILL.md"
+  if [[ ! -f "$skill_file" ]]; then
+    echo "Missing SKILL.md: $skill_file" >&2
+    status=1
+    continue
+  fi
+
+  if ! grep -Fq "Robustness Coverage (Known-Issue Matrix)" "$skill_file"; then
+    echo "[$skill] missing robustness coverage section" >&2
+    status=1
+  fi
+
+  if ! grep -Fq "$matrix_ref" "$skill_file"; then
+    echo "[$skill] missing matrix reference: $matrix_ref" >&2
+    status=1
+  fi
+
+  for issue_id in $expected; do
+    if ! grep -Fq "$issue_id" "$skill_file"; then
+      echo "[$skill] missing issue id: $issue_id" >&2
+      status=1
+    fi
+  done
+
+done
+
+if [[ "$target" != "all" && $found_target -eq 0 ]]; then
+  echo "Unknown skill: $target" >&2
+  usage
+  exit 2
+fi
+
+if [[ $status -ne 0 ]]; then
+  exit $status
+fi
+
+echo "Robustness audit checks passed for ${target}."

package/skills/opendevbrowser-best-practices/scripts/validate-skill-assets.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+skill_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+skill_file="$skill_root/SKILL.md"
+
+required_paths=(
+  "artifacts/provider-workflows.md"
+  "artifacts/parity-gates.md"
+  "artifacts/debug-trace-playbook.md"
+  "artifacts/fingerprint-tiers.md"
+  "artifacts/macro-workflows.md"
+  "artifacts/browser-agent-known-issues-matrix.md"
+  "artifacts/command-channel-reference.md"
+  "assets/templates/mode-flag-matrix.json"
+  "assets/templates/ops-request-envelope.json"
+  "assets/templates/cdp-forward-envelope.json"
+  "assets/templates/robustness-checklist.json"
+  "assets/templates/surface-audit-checklist.json"
+  "scripts/odb-workflow.sh"
+  "scripts/run-robustness-audit.sh"
+  "scripts/validate-skill-assets.sh"
+)
+
+json_templates=(
+  "assets/templates/mode-flag-matrix.json"
+  "assets/templates/ops-request-envelope.json"
+  "assets/templates/cdp-forward-envelope.json"
+  "assets/templates/robustness-checklist.json"
+  "assets/templates/surface-audit-checklist.json"
+)
+
+command_ref_markers=(
+  'CLI commands: `55`'
+  'Plugin tools: `48`'
+  '`/ops` command names: `38`'
+  'docs/SURFACE_REFERENCE.md'
+  'opencode'
+  'codex'
+  'claudecode'
+  'ampcli'
+)
+
+status=0
+
+for rel_path in "${required_paths[@]}"; do
+  full_path="$skill_root/$rel_path"
+  if [[ ! -f "$full_path" ]]; then
+    echo "Missing required asset: $rel_path" >&2
+    status=1
+  fi
+done
+
+for rel_path in "scripts/odb-workflow.sh" "scripts/run-robustness-audit.sh" "scripts/validate-skill-assets.sh"; do
+  full_path="$skill_root/$rel_path"
+  if [[ -f "$full_path" && ! -x "$full_path" ]]; then
+    echo "Script is not executable: $rel_path" >&2
+    status=1
+  fi
+done
+
+for rel_path in "${json_templates[@]}"; do
+  full_path="$skill_root/$rel_path"
+  if [[ -f "$full_path" ]]; then
+    if ! node -e 'const fs=require("fs"); JSON.parse(fs.readFileSync(process.argv[1], "utf8"));' "$full_path" >/dev/null 2>&1; then
+      echo "Invalid JSON template: $rel_path" >&2
+      status=1
+    fi
+  fi
+done
+
+for rel_path in "${required_paths[@]}"; do
+  if ! grep -Fq "$rel_path" "$skill_file"; then
+    echo "SKILL.md missing reference: $rel_path" >&2
+    status=1
+  fi
+done
+
+command_ref_path="$skill_root/artifacts/command-channel-reference.md"
+if [[ -f "$command_ref_path" ]]; then
+  for marker in "${command_ref_markers[@]}"; do
+    if ! grep -Fq "$marker" "$command_ref_path"; then
+      echo "Command/channel reference missing marker: $marker" >&2
+      status=1
+    fi
+  done
+fi
+
+if [[ $status -ne 0 ]]; then
+  exit $status
+fi
+
+echo "Skill assets validated: ${#required_paths[@]} files referenced/present, ${#json_templates[@]} JSON templates parsed."

package/skills/opendevbrowser-continuity-ledger/SKILL.md

@@ -8,6 +8,16 @@ version: 1.1.0
 
 Use this guide to maintain compaction-safe project state in `CONTINUITY.md`.
 
+## When to Use This Skill
+
+Use this skill when any of these are true:
+
+- Work is multi-step or long-running.
+- Context compaction or session handoff is likely.
+- Multiple agents are coordinating the same task.
+
+Start with `opendevbrowser-best-practices` quick start for execution. Switch here once continuity tracking is required.
+
 ## Ownership Rules
 
 Apply these rules exactly:

package/skills/opendevbrowser-data-extraction/SKILL.md

@@ -0,0 +1,126 @@
+---
+name: opendevbrowser-data-extraction
+description: This skill should be used when the user asks to "extract data from a page", "scrape tables", "collect paginated results", "parse list/card content", or "export structured web data" with OpenDevBrowser.
+version: 2.0.0
+---
+
+# Data Extraction Skill
+
+Use this skill to extract structured, auditable datasets from dynamic pages with compliance-aware workflows.
+
+## Pack Contents
+
+- `artifacts/extraction-workflows.md`
+- `assets/templates/extraction-schema.json`
+- `assets/templates/pagination-state.json`
+- `assets/templates/quality-gates.json`
+- `assets/templates/compliance-checklist.md`
+- `scripts/run-extraction-workflow.sh`
+- `scripts/validate-skill-assets.sh`
+- Shared robustness matrix: `../opendevbrowser-best-practices/artifacts/browser-agent-known-issues-matrix.md`
+
+## Fast Start
+
+```bash
+./skills/opendevbrowser-data-extraction/scripts/validate-skill-assets.sh
+./skills/opendevbrowser-data-extraction/scripts/run-extraction-workflow.sh list
+./skills/opendevbrowser-data-extraction/scripts/run-extraction-workflow.sh pagination
+./skills/opendevbrowser-data-extraction/scripts/run-extraction-workflow.sh infinite-scroll
+```
+
+## Core Rules
+
+- Define schema before extraction.
+- Track provenance for each record (`source_url`, `provider`, `captured_at`, `page`).
+- Prefer embedded structured data (JSON-LD/microdata) where available.
+- Stop on sustained anti-bot pressure (repeated 403/429/challenge loops).
+- Honor `Retry-After` and preserve checkpoint state before retrying pagination.
+
+## Parallel Multitab Alignment
+
+- Apply shared concurrency policy from `../opendevbrowser-best-practices/SKILL.md` ("Parallel Operations").
+- Run extraction acceptance on `managed`, `extension`, and `cdpConnect` before claiming mode parity.
+- Keep one session per worker; avoid interleaving `target-use` streams inside a single session.
+
+## Robustness Coverage (Known-Issue Matrix)
+
+Matrix source: `../opendevbrowser-best-practices/artifacts/browser-agent-known-issues-matrix.md`
+
+- `ISSUE-01`: stale refs after dynamic content updates
+- `ISSUE-06`: 429/backoff and retry budgeting
+- `ISSUE-08`: blocked/restricted origins and policy checks
+- `ISSUE-09`: pagination drift, duplicate accumulation, terminal detection
+- `ISSUE-10`: locale/currency parsing consistency
+
+## Extraction Planning
+
+1. Define required fields and null policy.
+2. Snapshot and map refs to schema.
+3. Choose pagination strategy.
+4. Apply quality gates each page.
+
+```text
+opendevbrowser_snapshot sessionId="<session-id>" format="actionables"
+```
+
+## Structured Data First
+
+Attempt extraction in this order:
+1. JSON-LD product/article blocks
+2. semantic table/list/card DOM
+3. fallback text parsing
+
+```text
+opendevbrowser_dom_get_text sessionId="<session-id>" ref="<json-ld-ref>"
+opendevbrowser_dom_get_html sessionId="<session-id>" ref="<table-ref>"
+```
+
+## Pagination Patterns
+
+### Numbered/Next pagination
+
+```text
+opendevbrowser_click sessionId="<session-id>" ref="<next-ref>"
+opendevbrowser_wait sessionId="<session-id>" until="networkidle"
+opendevbrowser_snapshot sessionId="<session-id>" format="actionables"
+```
+
+### Infinite scroll
+
+```text
+opendevbrowser_scroll sessionId="<session-id>" dy=1000
+opendevbrowser_wait sessionId="<session-id>" until="networkidle"
+```
+
+### Load more
+
+```text
+opendevbrowser_click sessionId="<session-id>" ref="<load-more-ref>"
+opendevbrowser_wait sessionId="<session-id>" until="networkidle"
+```
+
+## Quality Gates
+
+Apply per page:
+- dedupe by stable key (URL or canonical ID)
+- null-rate check for required fields
+- count delta check (new records must increase)
+- consistency check for currency/units
+- max consecutive challenge/429 loops before stop
+
+Use `assets/templates/quality-gates.json`.
+
+## Compliance and Safety
+
+- Respect robots and site terms.
+- Use pacing; do not flood endpoints.
+- Treat robots as policy guidance, not auth.
+- Stop or back off on repeated 429/403 and challenges.
+
+## References
+
+- RFC 9309 (robots protocol): https://www.rfc-editor.org/rfc/rfc9309
+- Google robots docs: https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt
+- Schema.org Offer: https://schema.org/Offer
+- Google Product structured data: https://developers.google.com/search/docs/appearance/structured-data/product-snippet
+- Playwright best practices: https://playwright.dev/docs/best-practices

package/skills/opendevbrowser-data-extraction/artifacts/extraction-workflows.md

@@ -0,0 +1,31 @@
+# Extraction Workflows
+
+## List/Card workflow
+
+1. Snapshot actionables
+2. Map repeating nodes
+3. Extract title/price/link fields
+4. Normalize and dedupe records
+5. Record parse confidence and currency normalization status
+
+## Table workflow
+
+1. Wait for table visibility
+2. Extract table HTML
+3. Parse rows/cells in host process
+4. Validate schema completeness
+
+## Pagination workflow
+
+1. Extract current page
+2. Advance page
+3. Wait + snapshot
+4. Repeat until terminal condition
+5. Persist checkpoint state every page
+
+## Anti-bot pressure workflow
+
+1. Detect repeated 403/429/challenge
+2. Parse and honor `Retry-After` if present
+3. Slow down / cooldown and resume from last checkpoint
+4. Stop and report if pressure persists after bounded retries

package/skills/opendevbrowser-data-extraction/assets/templates/compliance-checklist.md

@@ -0,0 +1,7 @@
+# Compliance Checklist
+
+- robots policy reviewed
+- site terms reviewed for data-use constraints
+- extraction rate limited and paced
+- anti-bot pressure monitored (429/403/challenge)
+- sensitive data excluded from artifacts

package/skills/opendevbrowser-data-extraction/assets/templates/extraction-schema.json

@@ -0,0 +1,17 @@
+{
+  "dataset": "",
+  "required_fields": [
+    "title",
+    "url"
+  ],
+  "optional_fields": [
+    "price",
+    "rating",
+    "availability"
+  ],
+  "field_types": {
+    "title": "string",
+    "url": "string",
+    "price": "number"
+  }
+}

package/skills/opendevbrowser-data-extraction/assets/templates/pagination-state.json

@@ -0,0 +1,11 @@
+{
+  "strategy": "next",
+  "current_page": 1,
+  "pages_processed": 0,
+  "records_collected": 0,
+  "last_page_delta": 0,
+  "checkpoint_token": "",
+  "retry_after_seconds": 0,
+  "challenge_loops": 0,
+  "stopping_reason": ""
+}

package/skills/opendevbrowser-data-extraction/assets/templates/quality-gates.json

@@ -0,0 +1,10 @@
+{
+  "max_required_null_rate": 0.05,
+  "max_duplicate_rate": 0.02,
+  "require_positive_page_delta": true,
+  "max_consecutive_empty_pages": 2,
+  "max_consecutive_challenge_loops": 2,
+  "max_retries_per_page": 2,
+  "respect_retry_after": true,
+  "stop_on_repeated_anti_bot_signals": true
+}

package/skills/opendevbrowser-data-extraction/examples/sample-schema.json

@@ -0,0 +1,19 @@
+{
+  "dataset": "shopping_offers",
+  "required_fields": [
+    "title",
+    "url",
+    "price"
+  ],
+  "optional_fields": [
+    "rating",
+    "reviews_count",
+    "availability"
+  ],
+  "field_types": {
+    "title": "string",
+    "url": "string",
+    "price": "number",
+    "rating": "number"
+  }
+}

package/skills/opendevbrowser-data-extraction/scripts/run-extraction-workflow.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+print_help() {
+  cat <<'HELP'
+Data extraction workflow router
+
+Usage:
+  ./skills/opendevbrowser-data-extraction/scripts/run-extraction-workflow.sh <workflow>
+
+Workflows:
+  list
+  table
+  pagination
+  infinite-scroll
+  load-more
+  anti-bot-pressure
+  checkpoint-resume
+  list-workflows
+HELP
+}
+
+workflow="${1:-list-workflows}"
+
+case "$workflow" in
+  list-workflows)
+    print_help
+    ;;
+  list)
+    cat <<'FLOW'
+opendevbrowser_snapshot sessionId="<session-id>" format="actionables"
+opendevbrowser_dom_get_text sessionId="<session-id>" ref="<item-title-ref>"
+opendevbrowser_get_attr sessionId="<session-id>" ref="<item-link-ref>" name="href"
+FLOW
+    ;;
+  table)
+    cat <<'FLOW'
+opendevbrowser_wait sessionId="<session-id>" until="networkidle"
+opendevbrowser_snapshot sessionId="<session-id>" format="actionables"
+opendevbrowser_dom_get_html sessionId="<session-id>" ref="<table-ref>"
+FLOW
+    ;;
+  pagination)
+    cat <<'FLOW'
+opendevbrowser_click sessionId="<session-id>" ref="<next-ref>"
+opendevbrowser_wait sessionId="<session-id>" until="networkidle"
+opendevbrowser_snapshot sessionId="<session-id>" format="actionables"
+FLOW
+    ;;
+  infinite-scroll)
+    cat <<'FLOW'
+opendevbrowser_scroll sessionId="<session-id>" dy=1000
+opendevbrowser_wait sessionId="<session-id>" until="networkidle"
+opendevbrowser_snapshot sessionId="<session-id>" format="actionables"
+FLOW
+    ;;
+  load-more)
+    cat <<'FLOW'
+opendevbrowser_click sessionId="<session-id>" ref="<load-more-ref>"
+opendevbrowser_wait sessionId="<session-id>" until="networkidle"
+opendevbrowser_snapshot sessionId="<session-id>" format="actionables"
+FLOW
+    ;;
+  anti-bot-pressure)
+    cat <<'FLOW'
+opendevbrowser_network_poll sessionId="<session-id>" max=100
+# Stop or back off on repeated 403/429/challenge signals; honor Retry-After.
+FLOW
+    ;;
+  checkpoint-resume)
+    cat <<'FLOW'
+# Resume from stored pagination checkpoint token/page index.
+opendevbrowser_snapshot sessionId="<session-id>" format="outline"
+opendevbrowser_wait sessionId="<session-id>" until="networkidle"
+opendevbrowser_snapshot sessionId="<session-id>" format="actionables"
+FLOW
+    ;;
+  *)
+    echo "Unknown workflow: $workflow" >&2
+    print_help
+    exit 2
+    ;;
+esac

package/skills/opendevbrowser-data-extraction/scripts/validate-skill-assets.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+skill_file="$root/SKILL.md"
+
+required=(
+  "artifacts/extraction-workflows.md"
+  "assets/templates/extraction-schema.json"
+  "assets/templates/pagination-state.json"
+  "assets/templates/quality-gates.json"
+  "assets/templates/compliance-checklist.md"
+  "scripts/run-extraction-workflow.sh"
+  "scripts/validate-skill-assets.sh"
+)
+
+status=0
+for rel in "${required[@]}"; do
+  if [[ ! -f "$root/$rel" ]]; then
+    echo "Missing required asset: $rel" >&2
+    status=1
+  fi
+  if ! grep -Fq "$rel" "$skill_file"; then
+    echo "SKILL.md missing reference: $rel" >&2
+    status=1
+  fi
+done
+
+for rel in scripts/run-extraction-workflow.sh scripts/validate-skill-assets.sh; do
+  if [[ -f "$root/$rel" && ! -x "$root/$rel" ]]; then
+    echo "Script is not executable: $rel" >&2
+    status=1
+  fi
+done
+
+for rel in assets/templates/extraction-schema.json assets/templates/pagination-state.json assets/templates/quality-gates.json; do
+  if [[ -f "$root/$rel" ]]; then
+    if ! node -e 'const fs=require("fs"); JSON.parse(fs.readFileSync(process.argv[1], "utf8"));' "$root/$rel" >/dev/null 2>&1; then
+      echo "Invalid JSON template: $rel" >&2
+      status=1
+    fi
+  fi
+done
+
+if [[ $status -ne 0 ]]; then
+  exit $status
+fi
+
+echo "Data extraction skill assets validated."