opendevbrowser 0.0.15 → 0.0.17

package/skills/opendevbrowser-best-practices/scripts/run-robustness-audit.sh

@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+skills_root="$(cd "$root/.." && pwd)"
+matrix_ref="../opendevbrowser-best-practices/artifacts/browser-agent-known-issues-matrix.md"
+
+usage() {
+  cat <<'HELP'
+Run robustness audit checks across opendevbrowser workflow skill packs.
+
+Usage:
+  ./skills/opendevbrowser-best-practices/scripts/run-robustness-audit.sh [skill-name]
+
+Examples:
+  ./skills/opendevbrowser-best-practices/scripts/run-robustness-audit.sh
+  ./skills/opendevbrowser-best-practices/scripts/run-robustness-audit.sh canvas-pack
+  ./skills/opendevbrowser-best-practices/scripts/run-robustness-audit.sh opendevbrowser-shopping
+HELP
+}
+
+skills=(
+  "opendevbrowser-login-automation:ISSUE-01 ISSUE-02 ISSUE-03 ISSUE-04 ISSUE-05 ISSUE-06 ISSUE-07"
+  "opendevbrowser-form-testing:ISSUE-01 ISSUE-02 ISSUE-05 ISSUE-06 ISSUE-08"
+  "opendevbrowser-data-extraction:ISSUE-01 ISSUE-06 ISSUE-08 ISSUE-09 ISSUE-10"
+  "opendevbrowser-shopping:ISSUE-06 ISSUE-09 ISSUE-10 ISSUE-11 ISSUE-12"
+  "opendevbrowser-product-presentation-asset:ISSUE-10 ISSUE-11 ISSUE-12"
+  "opendevbrowser-research:ISSUE-06 ISSUE-09 ISSUE-10 ISSUE-12"
+)
+
+canvas_issue_ids=(
+  "CANVAS-01"
+  "CANVAS-02"
+  "CANVAS-03"
+  "CANVAS-04"
+  "CANVAS-05"
+  "CANVAS-06"
+  "CANVAS-07"
+)
+
+status=0
+target="${1:-all}"
+
+check_canvas_pack() {
+  local root="$1"
+  local status_ref="$2"
+  local playbook="$root/artifacts/canvas-governance-playbook.md"
+  local command_ref="$root/artifacts/command-channel-reference.md"
+  local skill_file="$root/SKILL.md"
+  local feedback_eval="$root/assets/templates/canvas-feedback-eval.json"
+  local blocker_checklist="$root/assets/templates/canvas-blocker-checklist.json"
+  local robustness_checklist="$root/assets/templates/robustness-checklist.json"
+
+  for file_path in "$playbook" "$command_ref" "$skill_file" "$feedback_eval" "$blocker_checklist" "$robustness_checklist"; do
+    if [[ ! -f "$file_path" ]]; then
+      echo "[canvas-pack] missing file: $file_path" >&2
+      printf -v "$status_ref" '%s' 1
+      return
+    fi
+  done
+
+  for marker in \
+    "Canvas Governance Handshake" \
+    "artifacts/canvas-governance-playbook.md" \
+    "assets/templates/canvas-feedback-eval.json"; do
+    if ! grep -Fq "$marker" "$skill_file"; then
+      echo "[canvas-pack] SKILL.md missing marker: $marker" >&2
+      printf -v "$status_ref" '%s' 1
+    fi
+  done
+
+  for marker in \
+    "canvas.session.open" \
+    "canvas.plan.set" \
+    "canvas.feedback.poll" \
+    "plan_required"; do
+    if ! grep -Fq "$marker" "$command_ref"; then
+      echo "[canvas-pack] command/channel reference missing marker: $marker" >&2
+      printf -v "$status_ref" '%s' 1
+    fi
+  done
+
+  for issue_id in "${canvas_issue_ids[@]}"; do
+    if ! grep -Fq "$issue_id" "$playbook"; then
+      echo "[canvas-pack] playbook missing issue id: $issue_id" >&2
+      printf -v "$status_ref" '%s' 1
+    fi
+  done
+
+  if ! node -e '
+    const fs = require("fs");
+    const [feedbackPath, blockerPath, robustnessPath, ...expected] = process.argv.slice(1);
+    const feedback = JSON.parse(fs.readFileSync(feedbackPath, "utf8"));
+    const blockers = JSON.parse(fs.readFileSync(blockerPath, "utf8"));
+    const robustness = JSON.parse(fs.readFileSync(robustnessPath, "utf8"));
+    const feedbackIds = new Set(feedback.requiredIssueClasses || []);
+    const blockerIds = new Set((blockers.blockers || []).map((entry) => entry.auditId));
+    const robustnessIds = new Set((((robustness.canvas || {}).issueClasses) || []).map((entry) => entry.auditId));
+    const missing = expected.filter((id) => !feedbackIds.has(id) || !blockerIds.has(id) || !robustnessIds.has(id));
+    if (missing.length > 0) {
+      console.error(`[canvas-pack] missing canvas audit ids in JSON templates: ${missing.join(", ")}`);
+      process.exit(1);
+    }
+  ' "$feedback_eval" "$blocker_checklist" "$robustness_checklist" "${canvas_issue_ids[@]}"; then
+    printf -v "$status_ref" '%s' 1
+  fi
+}
+
+if [[ "$target" == "-h" || "$target" == "--help" ]]; then
+  usage
+  exit 0
+fi
+
+found_target=0
+if [[ "$target" == "all" || "$target" == "canvas-pack" ]]; then
+  found_target=1
+  check_canvas_pack "$root" status
+fi
+
+for entry in "${skills[@]}"; do
+  skill="${entry%%:*}"
+  expected="${entry#*:}"
+  if [[ "$target" != "all" && "$target" != "canvas-pack" && "$skill" != "$target" ]]; then
+    continue
+  fi
+  found_target=1
+
+  skill_file="$skills_root/$skill/SKILL.md"
+  if [[ ! -f "$skill_file" ]]; then
+    echo "Missing SKILL.md: $skill_file" >&2
+    status=1
+    continue
+  fi
+
+  if ! grep -Fq "Robustness Coverage (Known-Issue Matrix)" "$skill_file"; then
+    echo "[$skill] missing robustness coverage section" >&2
+    status=1
+  fi
+
+  if ! grep -Fq "$matrix_ref" "$skill_file"; then
+    echo "[$skill] missing matrix reference: $matrix_ref" >&2
+    status=1
+  fi
+
+  for issue_id in $expected; do
+    if ! grep -Fq "$issue_id" "$skill_file"; then
+      echo "[$skill] missing issue id: $issue_id" >&2
+      status=1
+    fi
+  done
+
+done
+
+if [[ "$target" != "all" && $found_target -eq 0 ]]; then
+  echo "Unknown skill: $target" >&2
+  usage
+  exit 2
+fi
+
+if [[ $status -ne 0 ]]; then
+  exit $status
+fi
+
+echo "Robustness audit checks passed for ${target}."

package/skills/opendevbrowser-best-practices/scripts/validate-skill-assets.sh

@@ -0,0 +1,234 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+skill_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+skill_file="$skill_root/SKILL.md"
+
+node - "$skill_root" "$skill_file" <<'NODE'
+const fs = require("node:fs");
+const path = require("node:path");
+
+const [skillRoot, skillFile] = process.argv.slice(2);
+
+const requiredPaths = [
+  "artifacts/provider-workflows.md",
+  "artifacts/parity-gates.md",
+  "artifacts/debug-trace-playbook.md",
+  "artifacts/fingerprint-tiers.md",
+  "artifacts/macro-workflows.md",
+  "artifacts/browser-agent-known-issues-matrix.md",
+  "artifacts/command-channel-reference.md",
+  "artifacts/canvas-governance-playbook.md",
+  "assets/templates/mode-flag-matrix.json",
+  "assets/templates/ops-request-envelope.json",
+  "assets/templates/cdp-forward-envelope.json",
+  "assets/templates/robustness-checklist.json",
+  "assets/templates/surface-audit-checklist.json",
+  "assets/templates/canvas-handshake-example.json",
+  "assets/templates/canvas-generation-plan.v1.json",
+  "assets/templates/canvas-feedback-eval.json",
+  "assets/templates/canvas-blocker-checklist.json",
+  "scripts/odb-workflow.sh",
+  "scripts/run-robustness-audit.sh",
+  "scripts/validate-skill-assets.sh"
+];
+
+const jsonTemplates = [
+  "assets/templates/mode-flag-matrix.json",
+  "assets/templates/ops-request-envelope.json",
+  "assets/templates/cdp-forward-envelope.json",
+  "assets/templates/robustness-checklist.json",
+  "assets/templates/surface-audit-checklist.json",
+  "assets/templates/canvas-handshake-example.json",
+  "assets/templates/canvas-generation-plan.v1.json",
+  "assets/templates/canvas-feedback-eval.json",
+  "assets/templates/canvas-blocker-checklist.json"
+];
+
+const executableScripts = [
+  "scripts/odb-workflow.sh",
+  "scripts/run-robustness-audit.sh",
+  "scripts/validate-skill-assets.sh"
+];
+
+const skillDocMarkers = [
+  "npx opendevbrowser --help",
+  "npx opendevbrowser help",
+  "56 CLI commands, 49 tools, 38 `/ops` commands, 26 `/canvas` commands",
+  "mutationPolicy.allowedBeforePlan",
+  "canvas.code.bind",
+  "canvas.code.resolve",
+  "bound_app_runtime",
+  "annotate --stored"
+];
+
+const commandRefMarkers = [
+  "CLI commands: `56`",
+  "Plugin tools: `49`",
+  "`/ops` command names: `38`",
+  "`/canvas` command names: `26`",
+  "docs/SURFACE_REFERENCE.md",
+  "npx opendevbrowser help",
+  "canvas.session.open",
+  "canvas.session.attach",
+  "canvas.feedback.poll",
+  "canvas.code.bind",
+  "canvas.code.resolve",
+  "canvas.tab.sync",
+  "canvas.overlay.sync",
+  "feedback.heartbeat",
+  "governanceRequirements",
+  "generationPlanRequirements",
+  "mutationPolicy",
+  "code-sync",
+  "parity",
+  "plan_required",
+  "annotate --stored",
+  "storage",
+  "opencode",
+  "codex",
+  "claudecode",
+  "ampcli"
+];
+
+const workflowMarkers = [
+  "canvas-preflight",
+  "canvas-feedback-eval"
+];
+
+const canvasPlaybookMarkers = [
+  "canvas.session.open",
+  "canvas.plan.set",
+  "canvas.feedback.poll",
+  "CANVAS-01",
+  "CANVAS-07"
+];
+
+const canvasTemplateMarkers = [
+  "CANVAS-01",
+  "CANVAS-02",
+  "CANVAS-03",
+  "CANVAS-04",
+  "CANVAS-05",
+  "CANVAS-06",
+  "CANVAS-07"
+];
+
+const templateMarkerPaths = [
+  "assets/templates/canvas-feedback-eval.json",
+  "assets/templates/canvas-blocker-checklist.json",
+  "assets/templates/robustness-checklist.json"
+];
+
+const surfaceAuditPath = "assets/templates/surface-audit-checklist.json";
+const surfaceAuditMarkers = [
+  "\"cliCommands\": 56",
+  "\"tools\": 49",
+  "\"opsCommands\": 38",
+  "\"canvasCommands\": 26",
+  "Canvas command names documented",
+  "Canvas code-sync surface documented",
+  "Annotation send/copy semantics documented"
+];
+
+const failures = [];
+
+const readUtf8 = (relPath) => fs.readFileSync(path.join(skillRoot, relPath), "utf8");
+const hasMarker = (content, marker) => content.includes(marker);
+
+for (const relPath of requiredPaths) {
+  if (!fs.existsSync(path.join(skillRoot, relPath))) {
+    failures.push(`Missing required asset: ${relPath}`);
+  }
+}
+
+for (const relPath of executableScripts) {
+  const fullPath = path.join(skillRoot, relPath);
+  if (!fs.existsSync(fullPath)) continue;
+  const mode = fs.statSync(fullPath).mode & 0o111;
+  if (mode === 0) {
+    failures.push(`Script is not executable: ${relPath}`);
+  }
+}
+
+for (const relPath of jsonTemplates) {
+  const fullPath = path.join(skillRoot, relPath);
+  if (!fs.existsSync(fullPath)) continue;
+  try {
+    JSON.parse(fs.readFileSync(fullPath, "utf8"));
+  } catch {
+    failures.push(`Invalid JSON template: ${relPath}`);
+  }
+}
+
+const skillDoc = fs.readFileSync(skillFile, "utf8");
+for (const relPath of requiredPaths) {
+  if (!hasMarker(skillDoc, relPath)) {
+    failures.push(`SKILL.md missing reference: ${relPath}`);
+  }
+}
+for (const marker of skillDocMarkers) {
+  if (!hasMarker(skillDoc, marker)) {
+    failures.push(`SKILL.md missing marker: ${marker}`);
+  }
+}
+
+const commandRefPath = "artifacts/command-channel-reference.md";
+if (fs.existsSync(path.join(skillRoot, commandRefPath))) {
+  const commandRef = readUtf8(commandRefPath);
+  for (const marker of commandRefMarkers) {
+    if (!hasMarker(commandRef, marker)) {
+      failures.push(`Command/channel reference missing marker: ${marker}`);
+    }
+  }
+}
+
+const workflowPath = "scripts/odb-workflow.sh";
+if (fs.existsSync(path.join(skillRoot, workflowPath))) {
+  const workflow = readUtf8(workflowPath);
+  for (const marker of workflowMarkers) {
+    if (!hasMarker(workflow, marker)) {
+      failures.push(`Workflow router missing marker: ${marker}`);
+    }
+  }
+}
+
+const canvasPlaybookPath = "artifacts/canvas-governance-playbook.md";
+if (fs.existsSync(path.join(skillRoot, canvasPlaybookPath))) {
+  const canvasPlaybook = readUtf8(canvasPlaybookPath);
+  for (const marker of canvasPlaybookMarkers) {
+    if (!hasMarker(canvasPlaybook, marker)) {
+      failures.push(`Canvas playbook missing marker: ${marker}`);
+    }
+  }
+}
+
+if (fs.existsSync(path.join(skillRoot, surfaceAuditPath))) {
+  const surfaceAudit = readUtf8(surfaceAuditPath);
+  for (const marker of surfaceAuditMarkers) {
+    if (!hasMarker(surfaceAudit, marker)) {
+      failures.push(`Surface audit checklist missing marker: ${marker}`);
+    }
+  }
+}
+
+for (const relPath of templateMarkerPaths) {
+  const fullPath = path.join(skillRoot, relPath);
+  if (!fs.existsSync(fullPath)) continue;
+  const content = fs.readFileSync(fullPath, "utf8");
+  for (const marker of canvasTemplateMarkers) {
+    if (!hasMarker(content, marker)) {
+      failures.push(`Canvas template missing marker (${marker}): ${relPath}`);
+    }
+  }
+}
+
+if (failures.length > 0) {
+  for (const failure of failures) {
+    console.error(failure);
+  }
+  process.exit(1);
+}
+
+console.log(`Skill assets validated: ${requiredPaths.length} files referenced/present, ${jsonTemplates.length} JSON templates parsed.`);
+NODE

package/skills/opendevbrowser-continuity-ledger/SKILL.md

@@ -8,6 +8,16 @@ version: 1.1.0
 
 Use this guide to maintain compaction-safe project state in `CONTINUITY.md`.
 
+## When to Use This Skill
+
+Use this skill when any of these are true:
+
+- Work is multi-step or long-running.
+- Context compaction or session handoff is likely.
+- Multiple agents are coordinating the same task.
+
+Start with `opendevbrowser-best-practices` quick start for execution. Switch here once continuity tracking is required.
+
 ## Ownership Rules
 
 Apply these rules exactly:

package/skills/opendevbrowser-data-extraction/SKILL.md

@@ -0,0 +1,126 @@
+---
+name: opendevbrowser-data-extraction
+description: This skill should be used when the user asks to "extract data from a page", "scrape tables", "collect paginated results", "parse list/card content", or "export structured web data" with OpenDevBrowser.
+version: 2.0.0
+---
+
+# Data Extraction Skill
+
+Use this skill to extract structured, auditable datasets from dynamic pages with compliance-aware workflows.
+
+## Pack Contents
+
+- `artifacts/extraction-workflows.md`
+- `assets/templates/extraction-schema.json`
+- `assets/templates/pagination-state.json`
+- `assets/templates/quality-gates.json`
+- `assets/templates/compliance-checklist.md`
+- `scripts/run-extraction-workflow.sh`
+- `scripts/validate-skill-assets.sh`
+- Shared robustness matrix: `../opendevbrowser-best-practices/artifacts/browser-agent-known-issues-matrix.md`
+
+## Fast Start
+
+```bash
+./skills/opendevbrowser-data-extraction/scripts/validate-skill-assets.sh
+./skills/opendevbrowser-data-extraction/scripts/run-extraction-workflow.sh list
+./skills/opendevbrowser-data-extraction/scripts/run-extraction-workflow.sh pagination
+./skills/opendevbrowser-data-extraction/scripts/run-extraction-workflow.sh infinite-scroll
+```
+
+## Core Rules
+
+- Define schema before extraction.
+- Track provenance for each record (`source_url`, `provider`, `captured_at`, `page`).
+- Prefer embedded structured data (JSON-LD/microdata) where available.
+- Stop on sustained anti-bot pressure (repeated 403/429/challenge loops).
+- Honor `Retry-After` and preserve checkpoint state before retrying pagination.
+
+## Parallel Multitab Alignment
+
+- Apply shared concurrency policy from `../opendevbrowser-best-practices/SKILL.md` ("Parallel Operations").
+- Run extraction acceptance on `managed`, `extension`, and `cdpConnect` before claiming mode parity.
+- Keep one session per worker; avoid interleaving `target-use` streams inside a single session.
+
+## Robustness Coverage (Known-Issue Matrix)
+
+Matrix source: `../opendevbrowser-best-practices/artifacts/browser-agent-known-issues-matrix.md`
+
+- `ISSUE-01`: stale refs after dynamic content updates
+- `ISSUE-06`: 429/backoff and retry budgeting
+- `ISSUE-08`: blocked/restricted origins and policy checks
+- `ISSUE-09`: pagination drift, duplicate accumulation, terminal detection
+- `ISSUE-10`: locale/currency parsing consistency
+
+## Extraction Planning
+
+1. Define required fields and null policy.
+2. Snapshot and map refs to schema.
+3. Choose pagination strategy.
+4. Apply quality gates each page.
+
+```text
+opendevbrowser_snapshot sessionId="<session-id>" format="actionables"
+```
+
+## Structured Data First
+
+Attempt extraction in this order:
+1. JSON-LD product/article blocks
+2. semantic table/list/card DOM
+3. fallback text parsing
+
+```text
+opendevbrowser_dom_get_text sessionId="<session-id>" ref="<json-ld-ref>"
+opendevbrowser_dom_get_html sessionId="<session-id>" ref="<table-ref>"
+```
+
+## Pagination Patterns
+
+### Numbered/Next pagination
+
+```text
+opendevbrowser_click sessionId="<session-id>" ref="<next-ref>"
+opendevbrowser_wait sessionId="<session-id>" until="networkidle"
+opendevbrowser_snapshot sessionId="<session-id>" format="actionables"
+```
+
+### Infinite scroll
+
+```text
+opendevbrowser_scroll sessionId="<session-id>" dy=1000
+opendevbrowser_wait sessionId="<session-id>" until="networkidle"
+```
+
+### Load more
+
+```text
+opendevbrowser_click sessionId="<session-id>" ref="<load-more-ref>"
+opendevbrowser_wait sessionId="<session-id>" until="networkidle"
+```
+
+## Quality Gates
+
+Apply per page:
+- dedupe by stable key (URL or canonical ID)
+- null-rate check for required fields
+- count delta check (new records must increase)
+- consistency check for currency/units
+- max consecutive challenge/429 loops before stop
+
+Use `assets/templates/quality-gates.json`.
+
+## Compliance and Safety
+
+- Respect robots and site terms.
+- Use pacing; do not flood endpoints.
+- Treat robots as policy guidance, not auth.
+- Stop or back off on repeated 429/403 and challenges.
+
+## References
+
+- RFC 9309 (robots protocol): https://www.rfc-editor.org/rfc/rfc9309
+- Google robots docs: https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt
+- Schema.org Offer: https://schema.org/Offer
+- Google Product structured data: https://developers.google.com/search/docs/appearance/structured-data/product-snippet
+- Playwright best practices: https://playwright.dev/docs/best-practices

package/skills/opendevbrowser-data-extraction/artifacts/extraction-workflows.md

@@ -0,0 +1,31 @@
+# Extraction Workflows
+
+## List/Card workflow
+
+1. Snapshot actionables
+2. Map repeating nodes
+3. Extract title/price/link fields
+4. Normalize and dedupe records
+5. Record parse confidence and currency normalization status
+
+## Table workflow
+
+1. Wait for table visibility
+2. Extract table HTML
+3. Parse rows/cells in host process
+4. Validate schema completeness
+
+## Pagination workflow
+
+1. Extract current page
+2. Advance page
+3. Wait + snapshot
+4. Repeat until terminal condition
+5. Persist checkpoint state every page
+
+## Anti-bot pressure workflow
+
+1. Detect repeated 403/429/challenge
+2. Parse and honor `Retry-After` if present
+3. Slow down / cooldown and resume from last checkpoint
+4. Stop and report if pressure persists after bounded retries

package/skills/opendevbrowser-data-extraction/assets/templates/compliance-checklist.md

@@ -0,0 +1,7 @@
+# Compliance Checklist
+
+- robots policy reviewed
+- site terms reviewed for data-use constraints
+- extraction rate limited and paced
+- anti-bot pressure monitored (429/403/challenge)
+- sensitive data excluded from artifacts

package/skills/opendevbrowser-data-extraction/assets/templates/extraction-schema.json

@@ -0,0 +1,17 @@
+{
+  "dataset": "",
+  "required_fields": [
+    "title",
+    "url"
+  ],
+  "optional_fields": [
+    "price",
+    "rating",
+    "availability"
+  ],
+  "field_types": {
+    "title": "string",
+    "url": "string",
+    "price": "number"
+  }
+}

package/skills/opendevbrowser-data-extraction/assets/templates/pagination-state.json

@@ -0,0 +1,11 @@
+{
+  "strategy": "next",
+  "current_page": 1,
+  "pages_processed": 0,
+  "records_collected": 0,
+  "last_page_delta": 0,
+  "checkpoint_token": "",
+  "retry_after_seconds": 0,
+  "challenge_loops": 0,
+  "stopping_reason": ""
+}

package/skills/opendevbrowser-data-extraction/assets/templates/quality-gates.json

@@ -0,0 +1,10 @@
+{
+  "max_required_null_rate": 0.05,
+  "max_duplicate_rate": 0.02,
+  "require_positive_page_delta": true,
+  "max_consecutive_empty_pages": 2,
+  "max_consecutive_challenge_loops": 2,
+  "max_retries_per_page": 2,
+  "respect_retry_after": true,
+  "stop_on_repeated_anti_bot_signals": true
+}

package/skills/opendevbrowser-data-extraction/examples/sample-schema.json

@@ -0,0 +1,19 @@
+{
+  "dataset": "shopping_offers",
+  "required_fields": [
+    "title",
+    "url",
+    "price"
+  ],
+  "optional_fields": [
+    "rating",
+    "reviews_count",
+    "availability"
+  ],
+  "field_types": {
+    "title": "string",
+    "url": "string",
+    "price": "number",
+    "rating": "number"
+  }
+}