mindforge-cc 11.8.2 → 11.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agent/mindforge/wf-orchestrate-review.md +35 -0
- package/.agent/mindforge/wf-verification-loop.md +45 -0
- package/.agent/mindforge/wf-workflow-optimizer.md +51 -0
- package/.agent/skills/article-illustrator/SKILL.md +7 -7
- package/.agent/skills/arxiv/SKILL.md +6 -6
- package/.agent/skills/code-wiki/SKILL.md +11 -11
- package/.agent/skills/concept-diagrams/SKILL.md +6 -6
- package/.agent/skills/concept-diagrams/examples/apartment-floor-plan-conversion.md +244 -0
- package/.agent/skills/concept-diagrams/examples/automated-password-reset-flow.md +276 -0
- package/.agent/skills/concept-diagrams/examples/autonomous-llm-research-agent-flow.md +240 -0
- package/.agent/skills/concept-diagrams/examples/banana-journey-tree-to-smoothie.md +161 -0
- package/.agent/skills/concept-diagrams/examples/commercial-aircraft-structure.md +209 -0
- package/.agent/skills/concept-diagrams/examples/cpu-ooo-microarchitecture.md +236 -0
- package/.agent/skills/concept-diagrams/examples/electricity-grid-flow.md +182 -0
- package/.agent/skills/concept-diagrams/examples/feature-film-production-pipeline.md +172 -0
- package/.agent/skills/concept-diagrams/examples/hospital-emergency-department-flow.md +165 -0
- package/.agent/skills/concept-diagrams/examples/ml-benchmark-grouped-bar-chart.md +114 -0
- package/.agent/skills/concept-diagrams/examples/place-order-uml-sequence.md +325 -0
- package/.agent/skills/concept-diagrams/examples/smart-city-infrastructure.md +173 -0
- package/.agent/skills/concept-diagrams/examples/smartphone-layer-anatomy.md +154 -0
- package/.agent/skills/concept-diagrams/examples/sn2-reaction-mechanism.md +247 -0
- package/.agent/skills/concept-diagrams/examples/wind-turbine-structure.md +338 -0
- package/.agent/skills/concept-diagrams/templates/template.html +174 -0
- package/.agent/skills/domain-intel/SKILL.md +7 -7
- package/.agent/skills/duckduckgo-search/SKILL.md +17 -17
- package/.agent/skills/github-auth/scripts/gh-env.sh +66 -0
- package/.agent/skills/github-code-review/SKILL.md +9 -9
- package/.agent/skills/github-pr-workflow/SKILL.md +3 -3
- package/.agent/skills/godmode/SKILL.md +8 -8
- package/.agent/skills/jupyter-live-kernel/SKILL.md +3 -3
- package/.agent/skills/kanban-orchestrator/SKILL.md +10 -10
- package/.agent/skills/kanban-worker/SKILL.md +6 -6
- package/.agent/skills/llm-wiki/SKILL.md +7 -7
- package/.agent/skills/mindforge-autonomous/SKILL.md +1 -1
- package/.agent/skills/mindforge-complete-milestone/SKILL.md +1 -1
- package/.agent/skills/mindforge-discuss-phase/SKILL.md +1 -1
- package/.agent/skills/mindforge-do/SKILL.md +1 -1
- package/.agent/skills/mindforge-execute-phase/SKILL.md +1 -1
- package/.agent/skills/mindforge-list-workspaces/SKILL.md +1 -1
- package/.agent/skills/mindforge-manager/SKILL.md +1 -1
- package/.agent/skills/mindforge-neural-orchestrator/SKILL.md +1 -1
- package/.agent/skills/mindforge-new-milestone/SKILL.md +4 -4
- package/.agent/skills/mindforge-new-project/SKILL.md +4 -4
- package/.agent/skills/mindforge-new-workspace/SKILL.md +1 -1
- package/.agent/skills/mindforge-note/SKILL.md +1 -1
- package/.agent/skills/mindforge-plan-phase/SKILL.md +1 -1
- package/.agent/skills/mindforge-profile-user/SKILL.md +1 -1
- package/.agent/skills/mindforge-remove-workspace/SKILL.md +1 -1
- package/.agent/skills/mindforge-research-phase/SKILL.md +1 -1
- package/.agent/skills/mindforge-ui-phase/SKILL.md +1 -1
- package/.agent/skills/mindforge-ui-review/SKILL.md +1 -1
- package/.agent/skills/mindforge-verify-work/SKILL.md +1 -1
- package/.agent/skills/mindforge-workspace-isolated/SKILL.md +1 -1
- package/.agent/skills/node-inspect-debugger/SKILL.md +1 -1
- package/.agent/skills/obsidian/SKILL.md +15 -15
- package/.agent/skills/osint-investigation/SKILL.md +3 -2
- package/.agent/skills/osint-investigation/references/sources/courtlistener.md +98 -0
- package/.agent/skills/osint-investigation/references/sources/gdelt.md +104 -0
- package/.agent/skills/osint-investigation/references/sources/icij-offshore.md +104 -0
- package/.agent/skills/osint-investigation/references/sources/nyc-acris.md +90 -0
- package/.agent/skills/osint-investigation/references/sources/ofac-sdn.md +92 -0
- package/.agent/skills/osint-investigation/references/sources/opencorporates.md +103 -0
- package/.agent/skills/osint-investigation/references/sources/sec-edgar.md +83 -0
- package/.agent/skills/osint-investigation/references/sources/senate-ld.md +89 -0
- package/.agent/skills/osint-investigation/references/sources/usaspending.md +97 -0
- package/.agent/skills/osint-investigation/references/sources/wayback.md +93 -0
- package/.agent/skills/osint-investigation/references/sources/wikipedia.md +107 -0
- package/.agent/skills/osint-investigation/scripts/_http.py +82 -0
- package/.agent/skills/osint-investigation/scripts/_normalize.py +67 -0
- package/.agent/skills/osint-investigation/scripts/build_findings.py +221 -0
- package/.agent/skills/osint-investigation/scripts/entity_resolution.py +228 -0
- package/.agent/skills/osint-investigation/scripts/fetch_courtlistener.py +149 -0
- package/.agent/skills/osint-investigation/scripts/fetch_gdelt.py +161 -0
- package/.agent/skills/osint-investigation/scripts/fetch_icij_offshore.py +234 -0
- package/.agent/skills/osint-investigation/scripts/fetch_nyc_acris.py +203 -0
- package/.agent/skills/osint-investigation/scripts/fetch_ofac_sdn.py +175 -0
- package/.agent/skills/osint-investigation/scripts/fetch_opencorporates.py +191 -0
- package/.agent/skills/osint-investigation/scripts/fetch_sec_edgar.py +184 -0
- package/.agent/skills/osint-investigation/scripts/fetch_senate_ld.py +146 -0
- package/.agent/skills/osint-investigation/scripts/fetch_usaspending.py +170 -0
- package/.agent/skills/osint-investigation/scripts/fetch_wayback.py +142 -0
- package/.agent/skills/osint-investigation/scripts/fetch_wikipedia.py +266 -0
- package/.agent/skills/osint-investigation/scripts/timing_analysis.py +252 -0
- package/.agent/skills/oss-forensics/SKILL.md +10 -9
- package/.agent/skills/oss-forensics/scripts/evidence-store.py +313 -0
- package/.agent/skills/parallel-cli/SKILL.md +2 -2
- package/.agent/skills/pinggy-tunnel/SKILL.md +3 -3
- package/.agent/skills/pixel-art/SKILL.md +2 -2
- package/.agent/skills/plan/SKILL.md +10 -10
- package/.agent/skills/requesting-code-review/SKILL.md +4 -6
- package/.agent/skills/research-paper-writing/SKILL.md +32 -32
- package/.agent/skills/rest-graphql-debug/SKILL.md +31 -32
- package/.agent/skills/s6-container-supervision/SKILL.md +1 -1
- package/.agent/skills/scrapling/SKILL.md +1 -1
- package/.agent/skills/sherlock/SKILL.md +1 -1
- package/.agent/skills/simplify-code/SKILL.md +5 -7
- package/.agent/skills/skill-authoring/SKILL.md +4 -4
- package/.agent/skills/spike/SKILL.md +11 -13
- package/.agent/skills/subagent-driven-development/SKILL.md +7 -11
- package/.agent/skills/systematic-debugging/SKILL.md +13 -14
- package/.agent/skills/test-driven-development/SKILL.md +6 -7
- package/.agent/skills/video-orchestrator/SKILL.md +1 -1
- package/.agent/skills/web-pentest/SKILL.md +7 -7
- package/.mindforge/config.json +3 -2
- package/.mindforge/dynamic-workflows/REGISTRY.md +4 -1
- package/.mindforge/dynamic-workflows/index.json +910 -840
- package/.mindforge/dynamic-workflows/scripts/accessibility-audit.js +88 -90
- package/.mindforge/dynamic-workflows/scripts/ai-model-eval.js +59 -61
- package/.mindforge/dynamic-workflows/scripts/api-contract-test.js +87 -85
- package/.mindforge/dynamic-workflows/scripts/api-migration.js +120 -122
- package/.mindforge/dynamic-workflows/scripts/architecture-modernization.js +84 -86
- package/.mindforge/dynamic-workflows/scripts/code-audit.js +75 -77
- package/.mindforge/dynamic-workflows/scripts/code-explainer.js +99 -101
- package/.mindforge/dynamic-workflows/scripts/competitive-analysis.js +62 -64
- package/.mindforge/dynamic-workflows/scripts/competitive-teardown.js +106 -108
- package/.mindforge/dynamic-workflows/scripts/cost-analysis.js +78 -80
- package/.mindforge/dynamic-workflows/scripts/data-pipeline-validate.js +47 -49
- package/.mindforge/dynamic-workflows/scripts/database-migration.js +85 -87
- package/.mindforge/dynamic-workflows/scripts/debug-detective.js +97 -99
- package/.mindforge/dynamic-workflows/scripts/dependency-health.js +81 -83
- package/.mindforge/dynamic-workflows/scripts/design-system-audit.js +84 -86
- package/.mindforge/dynamic-workflows/scripts/documentation-gen.js +65 -67
- package/.mindforge/dynamic-workflows/scripts/feature-planner.js +78 -80
- package/.mindforge/dynamic-workflows/scripts/incident-response.js +81 -83
- package/.mindforge/dynamic-workflows/scripts/multi-repo-sync.js +49 -51
- package/.mindforge/dynamic-workflows/scripts/mutation-testing.js +113 -115
- package/.mindforge/dynamic-workflows/scripts/onboard-codebase.js +79 -81
- package/.mindforge/dynamic-workflows/scripts/orchestrate-review.js +115 -0
- package/.mindforge/dynamic-workflows/scripts/perf-optimize.js +97 -99
- package/.mindforge/dynamic-workflows/scripts/pr-review.js +61 -63
- package/.mindforge/dynamic-workflows/scripts/refactor-plan.js +88 -90
- package/.mindforge/dynamic-workflows/scripts/release-prep.js +79 -81
- package/.mindforge/dynamic-workflows/scripts/security-hardening.js +117 -119
- package/.mindforge/dynamic-workflows/scripts/security-threat-model.js +119 -121
- package/.mindforge/dynamic-workflows/scripts/tdd-sprint.js +80 -82
- package/.mindforge/dynamic-workflows/scripts/tech-evaluation.js +50 -52
- package/.mindforge/dynamic-workflows/scripts/test-coverage-gap.js +68 -70
- package/.mindforge/dynamic-workflows/scripts/ux-heuristic-audit.js +89 -91
- package/.mindforge/dynamic-workflows/scripts/verification-loop.js +138 -0
- package/.mindforge/dynamic-workflows/scripts/workflow-optimizer.js +136 -0
- package/.mindforge/dynamic-workflows/scripts/writer-reviewer.js +63 -65
- package/.mindforge/memory/sync-manifest.json +1 -1
- package/.mindforge/personas/advisor-researcher.md +2 -2
- package/.mindforge/personas/api-load-tester.md +1 -1
- package/.mindforge/personas/authentication-architect.md +1 -1
- package/.mindforge/personas/backup-recovery-specialist.md +1 -1
- package/.mindforge/personas/build-optimizer.md +1 -1
- package/.mindforge/personas/caching-strategist.md +1 -1
- package/.mindforge/personas/chaos-engineer.md +1 -1
- package/.mindforge/personas/compliance-auditor.md +1 -1
- package/.mindforge/personas/config-management-expert.md +1 -1
- package/.mindforge/personas/coverage-specialist.md +1 -1
- package/.mindforge/personas/debug-specialist.md +2 -2
- package/.mindforge/personas/debugger.md +2 -2
- package/.mindforge/personas/dependency-auditor.md +1 -1
- package/.mindforge/personas/developer.md +1 -1
- package/.mindforge/personas/devops-engineer.md +1 -1
- package/.mindforge/personas/executor.md +1 -1
- package/.mindforge/personas/go-specialist.md +1 -1
- package/.mindforge/personas/incident-commander.md +1 -1
- package/.mindforge/personas/java-specialist.md +1 -1
- package/.mindforge/personas/kubernetes-debugger.md +1 -1
- package/.mindforge/personas/logging-architect.md +1 -1
- package/.mindforge/personas/mf-planner.md +1 -1
- package/.mindforge/personas/mf-researcher.md +1 -1
- package/.mindforge/personas/mf-reviewer.md +1 -1
- package/.mindforge/personas/nyquist-auditor.md +1 -1
- package/.mindforge/personas/observability-engineer.md +1 -1
- package/.mindforge/personas/performance-optimizer.md +1 -1
- package/.mindforge/personas/phase-researcher.md +5 -5
- package/.mindforge/personas/planner.md +1 -1
- package/.mindforge/personas/project-researcher.md +2 -2
- package/.mindforge/personas/python-specialist.md +1 -1
- package/.mindforge/personas/qa-engineer.md +1 -1
- package/.mindforge/personas/release-manager.md +1 -1
- package/.mindforge/personas/research-agent.md +4 -4
- package/.mindforge/personas/roadmapper-extend.md +1 -1
- package/.mindforge/personas/rust-specialist.md +1 -1
- package/.mindforge/personas/security-reviewer.md +1 -1
- package/.mindforge/personas/tech-writer.md +1 -1
- package/.mindforge/personas/typescript-wizard.md +1 -1
- package/.mindforge/personas/ui-researcher.md +1 -1
- package/.mindforge/skills/accessibility/SKILL.md +3 -3
- package/.mindforge/skills/agent-architecture-audit/SKILL.md +11 -0
- package/.mindforge/skills/agent-loops/SKILL.md +1 -1
- package/.mindforge/skills/arxiv/SKILL.md +6 -6
- package/.mindforge/skills/arxiv/scripts/search_arxiv.py +114 -0
- package/.mindforge/skills/code-tour/SKILL.md +2 -2
- package/.mindforge/skills/code-wiki/SKILL.md +11 -11
- package/.mindforge/skills/code-wiki/templates/README.md +31 -0
- package/.mindforge/skills/code-wiki/templates/architecture.md +30 -0
- package/.mindforge/skills/code-wiki/templates/getting-started.md +47 -0
- package/.mindforge/skills/code-wiki/templates/module.md +38 -0
- package/.mindforge/skills/concept-diagrams/SKILL.md +6 -6
- package/.mindforge/skills/concept-diagrams/examples/apartment-floor-plan-conversion.md +244 -0
- package/.mindforge/skills/concept-diagrams/examples/automated-password-reset-flow.md +276 -0
- package/.mindforge/skills/concept-diagrams/examples/autonomous-llm-research-agent-flow.md +240 -0
- package/.mindforge/skills/concept-diagrams/examples/banana-journey-tree-to-smoothie.md +161 -0
- package/.mindforge/skills/concept-diagrams/examples/commercial-aircraft-structure.md +209 -0
- package/.mindforge/skills/concept-diagrams/examples/cpu-ooo-microarchitecture.md +236 -0
- package/.mindforge/skills/concept-diagrams/examples/electricity-grid-flow.md +182 -0
- package/.mindforge/skills/concept-diagrams/examples/feature-film-production-pipeline.md +172 -0
- package/.mindforge/skills/concept-diagrams/examples/hospital-emergency-department-flow.md +165 -0
- package/.mindforge/skills/concept-diagrams/examples/ml-benchmark-grouped-bar-chart.md +114 -0
- package/.mindforge/skills/concept-diagrams/examples/place-order-uml-sequence.md +325 -0
- package/.mindforge/skills/concept-diagrams/examples/smart-city-infrastructure.md +173 -0
- package/.mindforge/skills/concept-diagrams/examples/smartphone-layer-anatomy.md +154 -0
- package/.mindforge/skills/concept-diagrams/examples/sn2-reaction-mechanism.md +247 -0
- package/.mindforge/skills/concept-diagrams/examples/wind-turbine-structure.md +338 -0
- package/.mindforge/skills/concept-diagrams/references/dashboard-patterns.md +43 -0
- package/.mindforge/skills/concept-diagrams/references/infrastructure-patterns.md +144 -0
- package/.mindforge/skills/concept-diagrams/references/physical-shape-cookbook.md +42 -0
- package/.mindforge/skills/concept-diagrams/templates/template.html +174 -0
- package/.mindforge/skills/creative-ideation/references/full-prompt-library.md +110 -0
- package/.mindforge/skills/documentation/SKILL.md +1 -1
- package/.mindforge/skills/domain-intel/SKILL.md +7 -7
- package/.mindforge/skills/domain-intel/scripts/domain_intel.py +397 -0
- package/.mindforge/skills/duckduckgo-search/SKILL.md +17 -17
- package/.mindforge/skills/duckduckgo-search/scripts/duckduckgo.sh +28 -0
- package/.mindforge/skills/environment-management/SKILL.md +1 -1
- package/.mindforge/skills/github-code-review/SKILL.md +9 -9
- package/.mindforge/skills/github-code-review/references/review-output-template.md +74 -0
- package/.mindforge/skills/github-issues/templates/bug-report.md +35 -0
- package/.mindforge/skills/github-issues/templates/feature-request.md +31 -0
- package/.mindforge/skills/github-pr-workflow/SKILL.md +3 -3
- package/.mindforge/skills/github-pr-workflow/references/ci-troubleshooting.md +183 -0
- package/.mindforge/skills/github-pr-workflow/references/conventional-commits.md +71 -0
- package/.mindforge/skills/github-pr-workflow/templates/pr-body-bugfix.md +35 -0
- package/.mindforge/skills/github-pr-workflow/templates/pr-body-feature.md +33 -0
- package/.mindforge/skills/incident-response/SKILL.md +1 -1
- package/.mindforge/skills/jupyter-live-kernel/SKILL.md +3 -3
- package/.mindforge/skills/kanban-orchestrator/SKILL.md +10 -10
- package/.mindforge/skills/kanban-worker/SKILL.md +6 -6
- package/.mindforge/skills/meme-generation/SKILL.md +1 -1
- package/.mindforge/skills/meme-generation/scripts/generate_meme.py +470 -0
- package/.mindforge/skills/meme-generation/scripts/templates.json +97 -0
- package/.mindforge/skills/obsidian/SKILL.md +15 -15
- package/.mindforge/skills/orch-pipeline/SKILL.md +2 -2
- package/.mindforge/skills/osint-investigation/SKILL.md +3 -2
- package/.mindforge/skills/osint-investigation/references/sources/courtlistener.md +98 -0
- package/.mindforge/skills/osint-investigation/references/sources/gdelt.md +104 -0
- package/.mindforge/skills/osint-investigation/references/sources/icij-offshore.md +104 -0
- package/.mindforge/skills/osint-investigation/references/sources/nyc-acris.md +90 -0
- package/.mindforge/skills/osint-investigation/references/sources/ofac-sdn.md +92 -0
- package/.mindforge/skills/osint-investigation/references/sources/opencorporates.md +103 -0
- package/.mindforge/skills/osint-investigation/references/sources/sec-edgar.md +83 -0
- package/.mindforge/skills/osint-investigation/references/sources/senate-ld.md +89 -0
- package/.mindforge/skills/osint-investigation/references/sources/usaspending.md +97 -0
- package/.mindforge/skills/osint-investigation/references/sources/wayback.md +93 -0
- package/.mindforge/skills/osint-investigation/references/sources/wikipedia.md +107 -0
- package/.mindforge/skills/osint-investigation/scripts/_http.py +82 -0
- package/.mindforge/skills/osint-investigation/scripts/_normalize.py +67 -0
- package/.mindforge/skills/osint-investigation/scripts/build_findings.py +221 -0
- package/.mindforge/skills/osint-investigation/scripts/entity_resolution.py +228 -0
- package/.mindforge/skills/osint-investigation/scripts/fetch_courtlistener.py +149 -0
- package/.mindforge/skills/osint-investigation/scripts/fetch_gdelt.py +161 -0
- package/.mindforge/skills/osint-investigation/scripts/fetch_icij_offshore.py +234 -0
- package/.mindforge/skills/osint-investigation/scripts/fetch_nyc_acris.py +203 -0
- package/.mindforge/skills/osint-investigation/scripts/fetch_ofac_sdn.py +175 -0
- package/.mindforge/skills/osint-investigation/scripts/fetch_opencorporates.py +191 -0
- package/.mindforge/skills/osint-investigation/scripts/fetch_sec_edgar.py +184 -0
- package/.mindforge/skills/osint-investigation/scripts/fetch_senate_ld.py +146 -0
- package/.mindforge/skills/osint-investigation/scripts/fetch_usaspending.py +170 -0
- package/.mindforge/skills/osint-investigation/scripts/fetch_wayback.py +142 -0
- package/.mindforge/skills/osint-investigation/scripts/fetch_wikipedia.py +266 -0
- package/.mindforge/skills/osint-investigation/scripts/timing_analysis.py +252 -0
- package/.mindforge/skills/osint-investigation/templates/source-template.md +59 -0
- package/.mindforge/skills/oss-forensics/SKILL.md +6 -6
- package/.mindforge/skills/oss-forensics/references/evidence-types.md +89 -0
- package/.mindforge/skills/oss-forensics/references/github-archive-guide.md +184 -0
- package/.mindforge/skills/oss-forensics/references/investigation-templates.md +131 -0
- package/.mindforge/skills/oss-forensics/references/recovery-techniques.md +164 -0
- package/.mindforge/skills/oss-forensics/scripts/evidence-store.py +313 -0
- package/.mindforge/skills/oss-forensics/templates/forensic-report.md +151 -0
- package/.mindforge/skills/oss-forensics/templates/malicious-package-report.md +43 -0
- package/.mindforge/skills/pixel-art/SKILL.md +2 -2
- package/.mindforge/skills/pixel-art/references/palettes.md +49 -0
- package/.mindforge/skills/plan/SKILL.md +10 -10
- package/.mindforge/skills/requesting-code-review/SKILL.md +4 -6
- package/.mindforge/skills/research-paper-writing/SKILL.md +32 -32
- package/.mindforge/skills/research-paper-writing/references/autoreason-methodology.md +394 -0
- package/.mindforge/skills/research-paper-writing/references/checklists.md +434 -0
- package/.mindforge/skills/research-paper-writing/references/citation-workflow.md +563 -0
- package/.mindforge/skills/research-paper-writing/references/experiment-patterns.md +728 -0
- package/.mindforge/skills/research-paper-writing/references/human-evaluation.md +476 -0
- package/.mindforge/skills/research-paper-writing/references/paper-types.md +481 -0
- package/.mindforge/skills/research-paper-writing/references/reviewer-guidelines.md +433 -0
- package/.mindforge/skills/research-paper-writing/references/sources.md +191 -0
- package/.mindforge/skills/research-paper-writing/references/writing-guide.md +474 -0
- package/.mindforge/skills/research-paper-writing/templates/README.md +251 -0
- package/.mindforge/skills/scrapling/SKILL.md +1 -1
- package/.mindforge/skills/security-review/SKILL.md +1 -1
- package/.mindforge/skills/sherlock/SKILL.md +1 -1
- package/.mindforge/skills/simplify-code/SKILL.md +5 -7
- package/.mindforge/skills/spike/SKILL.md +11 -13
- package/.mindforge/skills/subagent-driven-development/SKILL.md +7 -11
- package/.mindforge/skills/subagent-driven-development/references/context-budget-discipline.md +53 -0
- package/.mindforge/skills/subagent-driven-development/references/gates-taxonomy.md +93 -0
- package/.mindforge/skills/systematic-debugging/SKILL.md +13 -14
- package/.mindforge/skills/test-driven-development/SKILL.md +7 -8
- package/.mindforge/skills/testing-standards/SKILL.md +1 -1
- package/.mindforge/skills/web-pentest/SKILL.md +2 -2
- package/.mindforge/skills/web-pentest/references/bypass-techniques.md +133 -0
- package/.mindforge/skills/web-pentest/references/exploitation-techniques.md +204 -0
- package/.mindforge/skills/web-pentest/references/scope-enforcement.md +110 -0
- package/.mindforge/skills/web-pentest/references/vuln-taxonomy.md +81 -0
- package/.mindforge/skills/web-pentest/templates/authorization.md +69 -0
- package/.mindforge/skills/web-pentest/templates/pentest-report.md +178 -0
- package/.mindforge/skills/writing-skills/SKILL.md +4 -3
- package/CHANGELOG.md +35 -0
- package/MINDFORGE.md +3 -3
- package/README.md +9 -7
- package/RELEASENOTES.md +65 -2
- package/SECURITY.md +5 -3
- package/bin/autonomous/auto-runner.js +5 -2
- package/bin/engine/skill-loader.js +4 -0
- package/bin/governance/rbac.js +4 -0
- package/bin/memory/eis-client.js +1 -0
- package/bin/mindforge-cli.js +5 -0
- package/bin/models/bedrock-provider.js +192 -0
- package/bin/models/model-client.js +19 -0
- package/bin/spawn-agent.js +9 -2
- package/docs/commands-reference.md +68 -6
- package/docs/faq.md +38 -1
- package/docs/getting-started.md +27 -12
- package/docs/sdk-reference.md +36 -3
- package/docs/troubleshooting.md +59 -1
- package/docs/user-guide.md +7 -4
- package/package.json +3 -3
- package/.claude/commands/mindforge/skill-tdd.md +0 -53
- package/.claude/commands/mindforge/skills-index.md +0 -118
- package/.claude/commands/mindforge/systematic-debug.md +0 -60
- package/.claude/commands/mindforge/wf-accessibility-audit.md +0 -31
- package/.claude/commands/mindforge/wf-ai-model-eval.md +0 -31
- package/.claude/commands/mindforge/wf-api-migration.md +0 -31
- package/.claude/commands/mindforge/wf-architecture-modernization.md +0 -32
- package/.claude/commands/mindforge/wf-catalog.md +0 -59
- package/.claude/commands/mindforge/wf-code-audit.md +0 -31
- package/.claude/commands/mindforge/wf-code-explainer.md +0 -31
- package/.claude/commands/mindforge/wf-competitive-analysis.md +0 -31
- package/.claude/commands/mindforge/wf-competitive-teardown.md +0 -31
- package/.claude/commands/mindforge/wf-cost-analysis.md +0 -31
- package/.claude/commands/mindforge/wf-data-pipeline-validate.md +0 -31
- package/.claude/commands/mindforge/wf-database-migration.md +0 -31
- package/.claude/commands/mindforge/wf-debug-detective.md +0 -32
- package/.claude/commands/mindforge/wf-design-system-audit.md +0 -31
- package/.claude/commands/mindforge/wf-documentation-gen.md +0 -31
- package/.claude/commands/mindforge/wf-feature-planner.md +0 -31
- package/.claude/commands/mindforge/wf-incident-response.md +0 -31
- package/.claude/commands/mindforge/wf-multi-repo-sync.md +0 -31
- package/.claude/commands/mindforge/wf-mutation-testing.md +0 -31
- package/.claude/commands/mindforge/wf-onboard-codebase.md +0 -31
- package/.claude/commands/mindforge/wf-perf-optimize.md +0 -31
- package/.claude/commands/mindforge/wf-pr-review.md +0 -31
- package/.claude/commands/mindforge/wf-refactor-plan.md +0 -31
- package/.claude/commands/mindforge/wf-release-prep.md +0 -31
- package/.claude/commands/mindforge/wf-security-threat-model.md +0 -31
- package/.claude/commands/mindforge/wf-tdd-sprint.md +0 -31
- package/.claude/commands/mindforge/wf-tech-evaluation.md +0 -31
- package/.claude/commands/mindforge/wf-ux-heuristic-audit.md +0 -31
- package/.claude/commands/mindforge/wf-writer-reviewer.md +0 -30
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Fetch OFAC SDN list (CSV format) and normalize.
|
|
3
|
+
|
|
4
|
+
Public endpoint: https://www.treasury.gov/ofac/downloads/sdn.csv
|
|
5
|
+
Format reference: https://ofac.treasury.gov/specially-designated-nationals-and-blocked-persons-list-sdn-human-readable-lists
|
|
6
|
+
|
|
7
|
+
The SDN CSV uses a specific 12-column format with no header row:
|
|
8
|
+
ent_num, sdn_name, sdn_type, program, title, call_sign, vess_type,
|
|
9
|
+
tonnage, grt, vess_flag, vess_owner, remarks
|
|
10
|
+
Address and AKA records live in separate files. We fetch all three and join.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
import csv
|
|
16
|
+
import io
|
|
17
|
+
import sys
|
|
18
|
+
from collections import defaultdict
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
sys.path.insert(0, str(Path(__file__).parent))
|
|
22
|
+
from _http import get # noqa: E402
|
|
23
|
+
|
|
24
|
+
SDN_URL = "https://www.treasury.gov/ofac/downloads/sdn.csv"
|
|
25
|
+
ADD_URL = "https://www.treasury.gov/ofac/downloads/add.csv"
|
|
26
|
+
ALT_URL = "https://www.treasury.gov/ofac/downloads/alt.csv"
|
|
27
|
+
|
|
28
|
+
SDN_COLS = [
|
|
29
|
+
"ent_num", "sdn_name", "sdn_type", "program", "title",
|
|
30
|
+
"call_sign", "vess_type", "tonnage", "grt", "vess_flag",
|
|
31
|
+
"vess_owner", "remarks",
|
|
32
|
+
]
|
|
33
|
+
ADD_COLS = [
|
|
34
|
+
"ent_num", "add_num", "address", "city_state_zip", "country", "add_remarks",
|
|
35
|
+
]
|
|
36
|
+
ALT_COLS = [
|
|
37
|
+
"ent_num", "alt_num", "alt_type", "alt_name", "alt_remarks",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
COLUMNS = [
|
|
41
|
+
"entity_id",
|
|
42
|
+
"name",
|
|
43
|
+
"entity_type",
|
|
44
|
+
"program_list",
|
|
45
|
+
"title",
|
|
46
|
+
"nationalities",
|
|
47
|
+
"aka_list",
|
|
48
|
+
"addresses",
|
|
49
|
+
"dob",
|
|
50
|
+
"pob",
|
|
51
|
+
"remarks",
|
|
52
|
+
"last_updated",
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
_TYPE_MAP = {
|
|
56
|
+
"individual": "individual",
|
|
57
|
+
"entity": "entity",
|
|
58
|
+
"vessel": "vessel",
|
|
59
|
+
"aircraft": "aircraft",
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _read_csv(url: str, columns: list[str]) -> list[dict[str, str]]:
|
|
64
|
+
body = get(url, timeout=60).decode("latin-1", errors="replace")
|
|
65
|
+
reader = csv.reader(io.StringIO(body))
|
|
66
|
+
out = []
|
|
67
|
+
for row in reader:
|
|
68
|
+
if not row:
|
|
69
|
+
continue
|
|
70
|
+
# Pad/truncate to expected width.
|
|
71
|
+
row = row[: len(columns)] + [""] * (len(columns) - len(row))
|
|
72
|
+
out.append(dict(zip(columns, row)))
|
|
73
|
+
return out
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _strip_quotes(s: str) -> str:
|
|
77
|
+
s = s.strip()
|
|
78
|
+
if s.startswith('"') and s.endswith('"'):
|
|
79
|
+
s = s[1:-1]
|
|
80
|
+
if s == "-0-":
|
|
81
|
+
return ""
|
|
82
|
+
return s
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def fetch(
|
|
86
|
+
program: str | None,
|
|
87
|
+
entity_type: str | None,
|
|
88
|
+
out_path: str,
|
|
89
|
+
) -> int:
|
|
90
|
+
sdn = _read_csv(SDN_URL, SDN_COLS)
|
|
91
|
+
addresses = _read_csv(ADD_URL, ADD_COLS)
|
|
92
|
+
akas = _read_csv(ALT_URL, ALT_COLS)
|
|
93
|
+
|
|
94
|
+
addr_by_ent: dict[str, list[str]] = defaultdict(list)
|
|
95
|
+
for a in addresses:
|
|
96
|
+
ent = _strip_quotes(a["ent_num"])
|
|
97
|
+
parts = [
|
|
98
|
+
_strip_quotes(a[c])
|
|
99
|
+
for c in ("address", "city_state_zip", "country")
|
|
100
|
+
if _strip_quotes(a[c])
|
|
101
|
+
]
|
|
102
|
+
if parts:
|
|
103
|
+
addr_by_ent[ent].append(", ".join(parts))
|
|
104
|
+
|
|
105
|
+
aka_by_ent: dict[str, list[str]] = defaultdict(list)
|
|
106
|
+
for k in akas:
|
|
107
|
+
ent = _strip_quotes(k["ent_num"])
|
|
108
|
+
name = _strip_quotes(k["alt_name"])
|
|
109
|
+
if name:
|
|
110
|
+
aka_by_ent[ent].append(name)
|
|
111
|
+
|
|
112
|
+
rows: list[dict[str, str]] = []
|
|
113
|
+
for r in sdn:
|
|
114
|
+
ent_num = _strip_quotes(r["ent_num"])
|
|
115
|
+
if not ent_num:
|
|
116
|
+
continue
|
|
117
|
+
sdn_type = _TYPE_MAP.get(_strip_quotes(r["sdn_type"]).lower(), _strip_quotes(r["sdn_type"]))
|
|
118
|
+
if entity_type and sdn_type != entity_type:
|
|
119
|
+
continue
|
|
120
|
+
progs = _strip_quotes(r["program"])
|
|
121
|
+
if program and program.upper() not in progs.upper().split(";"):
|
|
122
|
+
continue
|
|
123
|
+
remarks = _strip_quotes(r["remarks"])
|
|
124
|
+
# DOB / POB are commonly embedded in remarks for individuals.
|
|
125
|
+
dob = ""
|
|
126
|
+
pob = ""
|
|
127
|
+
if sdn_type == "individual" and remarks:
|
|
128
|
+
for chunk in remarks.split(";"):
|
|
129
|
+
ch = chunk.strip()
|
|
130
|
+
if ch.upper().startswith("DOB"):
|
|
131
|
+
dob = ch.split(maxsplit=1)[1] if " " in ch else ""
|
|
132
|
+
elif ch.upper().startswith("POB"):
|
|
133
|
+
pob = ch.split(maxsplit=1)[1] if " " in ch else ""
|
|
134
|
+
rows.append(
|
|
135
|
+
{
|
|
136
|
+
"entity_id": ent_num,
|
|
137
|
+
"name": _strip_quotes(r["sdn_name"]),
|
|
138
|
+
"entity_type": sdn_type,
|
|
139
|
+
"program_list": "; ".join(p.strip() for p in progs.split(";") if p.strip()),
|
|
140
|
+
"title": _strip_quotes(r["title"]),
|
|
141
|
+
"nationalities": "", # not in this CSV; available in XML format
|
|
142
|
+
"aka_list": "; ".join(aka_by_ent.get(ent_num, [])),
|
|
143
|
+
"addresses": "; ".join(addr_by_ent.get(ent_num, [])),
|
|
144
|
+
"dob": dob,
|
|
145
|
+
"pob": pob,
|
|
146
|
+
"remarks": remarks,
|
|
147
|
+
"last_updated": "",
|
|
148
|
+
}
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
Path(out_path).parent.mkdir(parents=True, exist_ok=True)
|
|
152
|
+
with open(out_path, "w", newline="", encoding="utf-8") as fh:
|
|
153
|
+
w = csv.DictWriter(fh, fieldnames=COLUMNS)
|
|
154
|
+
w.writeheader()
|
|
155
|
+
w.writerows(rows)
|
|
156
|
+
return len(rows)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def main() -> int:
|
|
160
|
+
p = argparse.ArgumentParser(description=__doc__)
|
|
161
|
+
p.add_argument("--program", help="Filter to specific sanctions program (e.g. SDGT, IRAN)")
|
|
162
|
+
p.add_argument(
|
|
163
|
+
"--entity-type",
|
|
164
|
+
choices=["individual", "entity", "vessel", "aircraft"],
|
|
165
|
+
help="Filter to a specific entity type",
|
|
166
|
+
)
|
|
167
|
+
p.add_argument("--out", required=True)
|
|
168
|
+
a = p.parse_args()
|
|
169
|
+
n = fetch(program=a.program, entity_type=a.entity_type, out_path=a.out)
|
|
170
|
+
print(f"Wrote {n} OFAC SDN rows to {a.out}")
|
|
171
|
+
return 0
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
if __name__ == "__main__":
|
|
175
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Search OpenCorporates company registry data.
|
|
3
|
+
|
|
4
|
+
OpenCorporates aggregates ~200M companies from 130+ jurisdictions. The
|
|
5
|
+
public API requires an API token (free tier: 500 calls/month). Set
|
|
6
|
+
OPENCORPORATES_API_TOKEN in env or pass --token.
|
|
7
|
+
|
|
8
|
+
Without a token, this script falls back to scraping the public HTML
|
|
9
|
+
search page (limited fields, more brittle, no jurisdiction filter).
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import csv
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import sys
|
|
18
|
+
import urllib.parse
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
sys.path.insert(0, str(Path(__file__).parent))
|
|
22
|
+
from _http import get, get_json # noqa: E402
|
|
23
|
+
|
|
24
|
+
API_URL = "https://api.opencorporates.com/v0.4/companies/search"
|
|
25
|
+
HTML_URL = "https://opencorporates.com/companies"
|
|
26
|
+
|
|
27
|
+
COLUMNS = [
|
|
28
|
+
"name",
|
|
29
|
+
"company_number",
|
|
30
|
+
"jurisdiction_code",
|
|
31
|
+
"jurisdiction_name",
|
|
32
|
+
"incorporation_date",
|
|
33
|
+
"dissolution_date",
|
|
34
|
+
"company_type",
|
|
35
|
+
"status",
|
|
36
|
+
"registered_address",
|
|
37
|
+
"opencorporates_url",
|
|
38
|
+
"officers_count",
|
|
39
|
+
"source",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _via_api(query: str, jurisdiction: str | None, token: str, limit: int) -> list[dict]:
|
|
44
|
+
params = {
|
|
45
|
+
"q": query,
|
|
46
|
+
"api_token": token,
|
|
47
|
+
"per_page": str(min(limit, 100)),
|
|
48
|
+
}
|
|
49
|
+
if jurisdiction:
|
|
50
|
+
params["jurisdiction_code"] = jurisdiction
|
|
51
|
+
url = f"{API_URL}?{urllib.parse.urlencode(params)}"
|
|
52
|
+
payload = get_json(url)
|
|
53
|
+
if not isinstance(payload, dict):
|
|
54
|
+
return []
|
|
55
|
+
results = payload.get("results", {}).get("companies", []) or []
|
|
56
|
+
return [r.get("company", {}) for r in results if isinstance(r, dict)]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _via_html(query: str, limit: int) -> list[dict]:
|
|
60
|
+
"""Best-effort HTML fallback when no API token is available."""
|
|
61
|
+
params = {"q": query, "utf8": "✓"}
|
|
62
|
+
url = f"{HTML_URL}?{urllib.parse.urlencode(params)}"
|
|
63
|
+
body = get(url, user_agent="Mozilla/5.0 hermes-osint").decode("utf-8", errors="replace")
|
|
64
|
+
# Each result is in <li class="company"> ... </li> with name, url, status
|
|
65
|
+
pattern = re.compile(
|
|
66
|
+
r'<li[^>]*class="[^"]*company[^"]*"[^>]*>.*?'
|
|
67
|
+
r'<a[^>]+href="(?P<url>/companies/[^"]+)"[^>]*>(?P<name>[^<]+)</a>'
|
|
68
|
+
r'(?:.*?<span[^>]*class="[^"]*jurisdiction[^"]*"[^>]*>(?P<jur>[^<]+)</span>)?'
|
|
69
|
+
r"(?:.*?<dt[^>]*>(?:Company\s+Number|Number)</dt>\s*<dd[^>]*>(?P<num>[^<]+)</dd>)?",
|
|
70
|
+
re.DOTALL | re.IGNORECASE,
|
|
71
|
+
)
|
|
72
|
+
out = []
|
|
73
|
+
for m in pattern.finditer(body):
|
|
74
|
+
if len(out) >= limit:
|
|
75
|
+
break
|
|
76
|
+
url_path = m.group("url").strip()
|
|
77
|
+
out.append(
|
|
78
|
+
{
|
|
79
|
+
"name": (m.group("name") or "").strip(),
|
|
80
|
+
"opencorporates_url": f"https://opencorporates.com{url_path}",
|
|
81
|
+
"jurisdiction_code": (m.group("jur") or "").strip(),
|
|
82
|
+
"company_number": (m.group("num") or "").strip(),
|
|
83
|
+
"_via": "html",
|
|
84
|
+
}
|
|
85
|
+
)
|
|
86
|
+
return out
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def fetch(
|
|
90
|
+
query: str,
|
|
91
|
+
jurisdiction: str | None,
|
|
92
|
+
token: str | None,
|
|
93
|
+
limit: int,
|
|
94
|
+
out_path: str,
|
|
95
|
+
) -> int:
|
|
96
|
+
if token:
|
|
97
|
+
try:
|
|
98
|
+
companies = _via_api(query, jurisdiction, token, limit)
|
|
99
|
+
source_tag = "api"
|
|
100
|
+
except Exception as e: # noqa: BLE001
|
|
101
|
+
print(
|
|
102
|
+
f"OpenCorporates API call failed ({e}); falling back to HTML.",
|
|
103
|
+
file=sys.stderr,
|
|
104
|
+
)
|
|
105
|
+
companies = _via_html(query, limit)
|
|
106
|
+
source_tag = "html-fallback"
|
|
107
|
+
else:
|
|
108
|
+
print(
|
|
109
|
+
"OPENCORPORATES_API_TOKEN not set — using HTML fallback (limited fields). "
|
|
110
|
+
"Get a free token at https://opencorporates.com/api_accounts/new",
|
|
111
|
+
file=sys.stderr,
|
|
112
|
+
)
|
|
113
|
+
companies = _via_html(query, limit)
|
|
114
|
+
source_tag = "html"
|
|
115
|
+
|
|
116
|
+
rows: list[dict[str, str]] = []
|
|
117
|
+
for c in companies[:limit]:
|
|
118
|
+
if c.get("_via") == "html":
|
|
119
|
+
rows.append(
|
|
120
|
+
{
|
|
121
|
+
"name": c.get("name", ""),
|
|
122
|
+
"company_number": c.get("company_number", ""),
|
|
123
|
+
"jurisdiction_code": c.get("jurisdiction_code", ""),
|
|
124
|
+
"jurisdiction_name": "",
|
|
125
|
+
"incorporation_date": "",
|
|
126
|
+
"dissolution_date": "",
|
|
127
|
+
"company_type": "",
|
|
128
|
+
"status": "",
|
|
129
|
+
"registered_address": "",
|
|
130
|
+
"opencorporates_url": c.get("opencorporates_url", ""),
|
|
131
|
+
"officers_count": "",
|
|
132
|
+
"source": source_tag,
|
|
133
|
+
}
|
|
134
|
+
)
|
|
135
|
+
continue
|
|
136
|
+
addr = c.get("registered_address_in_full") or ""
|
|
137
|
+
rows.append(
|
|
138
|
+
{
|
|
139
|
+
"name": c.get("name", "") or "",
|
|
140
|
+
"company_number": c.get("company_number", "") or "",
|
|
141
|
+
"jurisdiction_code": c.get("jurisdiction_code", "") or "",
|
|
142
|
+
"jurisdiction_name": "",
|
|
143
|
+
"incorporation_date": c.get("incorporation_date", "") or "",
|
|
144
|
+
"dissolution_date": c.get("dissolution_date", "") or "",
|
|
145
|
+
"company_type": c.get("company_type", "") or "",
|
|
146
|
+
"status": c.get("current_status", "") or c.get("inactive", "") or "",
|
|
147
|
+
"registered_address": addr,
|
|
148
|
+
"opencorporates_url": c.get("opencorporates_url", "") or "",
|
|
149
|
+
"officers_count": str(c.get("officers", {}).get("total_count", "") if c.get("officers") else ""),
|
|
150
|
+
"source": source_tag,
|
|
151
|
+
}
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
Path(out_path).parent.mkdir(parents=True, exist_ok=True)
|
|
155
|
+
with open(out_path, "w", newline="", encoding="utf-8") as fh:
|
|
156
|
+
w = csv.DictWriter(fh, fieldnames=COLUMNS)
|
|
157
|
+
w.writeheader()
|
|
158
|
+
w.writerows(rows)
|
|
159
|
+
if not rows:
|
|
160
|
+
print(
|
|
161
|
+
f"OpenCorporates: 0 matches for query={query!r}"
|
|
162
|
+
f"{f' jurisdiction={jurisdiction!r}' if jurisdiction else ''}.",
|
|
163
|
+
file=sys.stderr,
|
|
164
|
+
)
|
|
165
|
+
return len(rows)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def main() -> int:
|
|
169
|
+
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
170
|
+
p.add_argument("--query", required=True, help="Company name search")
|
|
171
|
+
p.add_argument(
|
|
172
|
+
"--jurisdiction",
|
|
173
|
+
help="Jurisdiction code, e.g. 'us_ny', 'us_de', 'gb', 'sg' (lowercased OpenCorporates style)",
|
|
174
|
+
)
|
|
175
|
+
p.add_argument("--limit", type=int, default=50)
|
|
176
|
+
p.add_argument("--token", default=os.environ.get("OPENCORPORATES_API_TOKEN"))
|
|
177
|
+
p.add_argument("--out", required=True)
|
|
178
|
+
a = p.parse_args()
|
|
179
|
+
n = fetch(
|
|
180
|
+
query=a.query,
|
|
181
|
+
jurisdiction=a.jurisdiction,
|
|
182
|
+
token=a.token,
|
|
183
|
+
limit=a.limit,
|
|
184
|
+
out_path=a.out,
|
|
185
|
+
)
|
|
186
|
+
print(f"Wrote {n} OpenCorporates rows to {a.out}")
|
|
187
|
+
return 0
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
if __name__ == "__main__":
|
|
191
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Fetch SEC EDGAR filings index for a given CIK or company name.
|
|
3
|
+
|
|
4
|
+
SEC requires a User-Agent header with contact info. Set SEC_USER_AGENT,
|
|
5
|
+
e.g. SEC_USER_AGENT="Research example@example.com".
|
|
6
|
+
|
|
7
|
+
Filings JSON is published at:
|
|
8
|
+
https://data.sec.gov/submissions/CIK<10-digit-padded>.json
|
|
9
|
+
|
|
10
|
+
Company lookup uses:
|
|
11
|
+
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&company=<name>&output=atom
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import csv
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
import sys
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
sys.path.insert(0, str(Path(__file__).parent))
|
|
23
|
+
from _http import get, get_json # noqa: E402
|
|
24
|
+
|
|
25
|
+
SUBMISSIONS_URL = "https://data.sec.gov/submissions/CIK{cik}.json"
|
|
26
|
+
COLUMNS = [
|
|
27
|
+
"cik",
|
|
28
|
+
"company_name",
|
|
29
|
+
"form_type",
|
|
30
|
+
"filing_date",
|
|
31
|
+
"accession_number",
|
|
32
|
+
"primary_document",
|
|
33
|
+
"filing_url",
|
|
34
|
+
"reporting_period",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _ua() -> str:
|
|
39
|
+
ua = os.environ.get("SEC_USER_AGENT", "").strip()
|
|
40
|
+
if not ua:
|
|
41
|
+
raise SystemExit(
|
|
42
|
+
"SEC requires a User-Agent with contact info. "
|
|
43
|
+
"Set SEC_USER_AGENT='Your Name your@email'."
|
|
44
|
+
)
|
|
45
|
+
return ua
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _resolve_cik(company: str) -> tuple[str, str]:
|
|
49
|
+
"""Resolve a company name to a CIK via EDGAR's atom feed.
|
|
50
|
+
|
|
51
|
+
Returns (cik, resolved_company_name). The feed entries also reveal whether
|
|
52
|
+
the match is an individual filer (Form 3/4/5 only) — surfaced in the
|
|
53
|
+
return value so callers can warn.
|
|
54
|
+
"""
|
|
55
|
+
url = "https://www.sec.gov/cgi-bin/browse-edgar"
|
|
56
|
+
params = {"action": "getcompany", "company": company, "output": "atom", "owner": "include"}
|
|
57
|
+
body = get(url, params=params, user_agent=_ua()).decode("utf-8", errors="replace")
|
|
58
|
+
m = re.search(r"CIK=(\d{10})", body)
|
|
59
|
+
if not m:
|
|
60
|
+
raise SystemExit(f"Could not resolve CIK for company={company!r}")
|
|
61
|
+
cik = m.group(1)
|
|
62
|
+
name_m = re.search(r"<title>([^<]+)\s*\((\d{10})\)</title>", body)
|
|
63
|
+
resolved = name_m.group(1).strip() if name_m else ""
|
|
64
|
+
return cik, resolved
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def fetch(
|
|
68
|
+
cik: str | None,
|
|
69
|
+
company: str | None,
|
|
70
|
+
types: list[str],
|
|
71
|
+
since: str | None,
|
|
72
|
+
out_path: str,
|
|
73
|
+
) -> int:
|
|
74
|
+
resolved_name = ""
|
|
75
|
+
if not cik and company:
|
|
76
|
+
try:
|
|
77
|
+
cik, resolved_name = _resolve_cik(company) # type: ignore[assignment]
|
|
78
|
+
except SystemExit as e:
|
|
79
|
+
# Write empty CSV with header so downstream tools still work,
|
|
80
|
+
# and tell the user clearly.
|
|
81
|
+
print(f"SEC EDGAR: {e}", file=sys.stderr)
|
|
82
|
+
Path(out_path).parent.mkdir(parents=True, exist_ok=True)
|
|
83
|
+
with open(out_path, "w", newline="", encoding="utf-8") as fh:
|
|
84
|
+
csv.DictWriter(fh, fieldnames=COLUMNS).writeheader()
|
|
85
|
+
return 0
|
|
86
|
+
if resolved_name:
|
|
87
|
+
print(
|
|
88
|
+
f"Resolved company={company!r} → CIK {cik} ({resolved_name})",
|
|
89
|
+
file=sys.stderr,
|
|
90
|
+
)
|
|
91
|
+
if not cik:
|
|
92
|
+
raise SystemExit("must supply --cik or --company")
|
|
93
|
+
cik = cik.zfill(10)
|
|
94
|
+
url = SUBMISSIONS_URL.format(cik=cik)
|
|
95
|
+
payload = get_json(url, user_agent=_ua())
|
|
96
|
+
if not isinstance(payload, dict):
|
|
97
|
+
raise SystemExit(f"Unexpected EDGAR response shape for CIK {cik}")
|
|
98
|
+
name = payload.get("name", "")
|
|
99
|
+
recent = (payload.get("filings", {}) or {}).get("recent", {}) or {}
|
|
100
|
+
form = recent.get("form", [])
|
|
101
|
+
date = recent.get("filingDate", [])
|
|
102
|
+
accession = recent.get("accessionNumber", [])
|
|
103
|
+
primary_doc = recent.get("primaryDocument", [])
|
|
104
|
+
period = recent.get("reportDate", [])
|
|
105
|
+
|
|
106
|
+
# Histogram of available filing types — useful for surfacing why a filter
|
|
107
|
+
# returned 0 (e.g. user asked for 10-K on an individual Form 4 filer).
|
|
108
|
+
type_hist: dict[str, int] = {}
|
|
109
|
+
for ftype in form:
|
|
110
|
+
type_hist[ftype] = type_hist.get(ftype, 0) + 1
|
|
111
|
+
|
|
112
|
+
type_set = {t.strip().upper() for t in types} if types else None
|
|
113
|
+
rows: list[dict[str, str]] = []
|
|
114
|
+
for i, ftype in enumerate(form):
|
|
115
|
+
if type_set and ftype.upper() not in type_set:
|
|
116
|
+
continue
|
|
117
|
+
fdate = date[i] if i < len(date) else ""
|
|
118
|
+
if since and fdate and fdate < since:
|
|
119
|
+
continue
|
|
120
|
+
acc = accession[i] if i < len(accession) else ""
|
|
121
|
+
pdoc = primary_doc[i] if i < len(primary_doc) else ""
|
|
122
|
+
acc_nodash = acc.replace("-", "")
|
|
123
|
+
filing_url = (
|
|
124
|
+
f"https://www.sec.gov/Archives/edgar/data/{int(cik)}/{acc_nodash}/{pdoc}"
|
|
125
|
+
if acc and pdoc
|
|
126
|
+
else ""
|
|
127
|
+
)
|
|
128
|
+
rows.append(
|
|
129
|
+
{
|
|
130
|
+
"cik": cik,
|
|
131
|
+
"company_name": name,
|
|
132
|
+
"form_type": ftype,
|
|
133
|
+
"filing_date": fdate,
|
|
134
|
+
"accession_number": acc,
|
|
135
|
+
"primary_document": pdoc,
|
|
136
|
+
"filing_url": filing_url,
|
|
137
|
+
"reporting_period": period[i] if i < len(period) else "",
|
|
138
|
+
}
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
Path(out_path).parent.mkdir(parents=True, exist_ok=True)
|
|
142
|
+
with open(out_path, "w", newline="", encoding="utf-8") as fh:
|
|
143
|
+
w = csv.DictWriter(fh, fieldnames=COLUMNS)
|
|
144
|
+
w.writeheader()
|
|
145
|
+
w.writerows(rows)
|
|
146
|
+
|
|
147
|
+
if not rows and type_hist:
|
|
148
|
+
top = sorted(type_hist.items(), key=lambda kv: -kv[1])[:8]
|
|
149
|
+
hist_str = ", ".join(f"{t}={n}" for t, n in top)
|
|
150
|
+
print(
|
|
151
|
+
f"Warning: SEC EDGAR CIK {cik} ({name}) has {sum(type_hist.values())} "
|
|
152
|
+
f"recent filings but NONE match types={types}. "
|
|
153
|
+
f"Available form types: {hist_str}.",
|
|
154
|
+
file=sys.stderr,
|
|
155
|
+
)
|
|
156
|
+
# Insider-filer heuristic: only Form 3/4/5 → individual person, not a company.
|
|
157
|
+
company_types = {"10-K", "10-Q", "8-K", "20-F", "DEF 14A", "S-1"}
|
|
158
|
+
if not (set(type_hist.keys()) & company_types):
|
|
159
|
+
print(
|
|
160
|
+
f"Note: CIK {cik} appears to be an INDIVIDUAL filer "
|
|
161
|
+
f"(insider Form 3/4/5 only), not a corporate registrant. "
|
|
162
|
+
f"The resolver may have matched an officer/director named "
|
|
163
|
+
f"{company!r} rather than a company.",
|
|
164
|
+
file=sys.stderr,
|
|
165
|
+
)
|
|
166
|
+
return len(rows)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def main() -> int:
|
|
170
|
+
p = argparse.ArgumentParser(description=__doc__)
|
|
171
|
+
p.add_argument("--cik", help="Central Index Key (will be 10-digit zero-padded)")
|
|
172
|
+
p.add_argument("--company", help="Resolve to CIK by company name")
|
|
173
|
+
p.add_argument("--types", default="", help="Comma-separated form types (e.g. 10-K,10-Q,8-K)")
|
|
174
|
+
p.add_argument("--since", help="Skip filings before YYYY-MM-DD")
|
|
175
|
+
p.add_argument("--out", required=True)
|
|
176
|
+
a = p.parse_args()
|
|
177
|
+
types = [t for t in (a.types or "").split(",") if t.strip()]
|
|
178
|
+
n = fetch(cik=a.cik, company=a.company, types=types, since=a.since, out_path=a.out)
|
|
179
|
+
print(f"Wrote {n} EDGAR filing rows to {a.out}")
|
|
180
|
+
return 0
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
if __name__ == "__main__":
|
|
184
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Fetch Senate Lobbying Disclosure (LD-1 / LD-2) filings.
|
|
3
|
+
|
|
4
|
+
Anonymous: 120 req/hour. Token (SENATE_LDA_TOKEN): 1200 req/hour.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import csv
|
|
10
|
+
import os
|
|
11
|
+
import sys
|
|
12
|
+
import time
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
sys.path.insert(0, str(Path(__file__).parent))
|
|
16
|
+
from _http import get_json # noqa: E402
|
|
17
|
+
|
|
18
|
+
ENDPOINT = "https://lda.senate.gov/api/v1/filings/"
|
|
19
|
+
COLUMNS = [
|
|
20
|
+
"filing_uuid",
|
|
21
|
+
"filing_type",
|
|
22
|
+
"filing_year",
|
|
23
|
+
"filing_period",
|
|
24
|
+
"registrant_name",
|
|
25
|
+
"registrant_id",
|
|
26
|
+
"client_name",
|
|
27
|
+
"client_id",
|
|
28
|
+
"client_general_description",
|
|
29
|
+
"income",
|
|
30
|
+
"expenses",
|
|
31
|
+
"lobbyists",
|
|
32
|
+
"issues",
|
|
33
|
+
"government_entities",
|
|
34
|
+
"filing_date",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def fetch(
|
|
39
|
+
client: str | None,
|
|
40
|
+
registrant: str | None,
|
|
41
|
+
year: int,
|
|
42
|
+
token: str | None,
|
|
43
|
+
out_path: str,
|
|
44
|
+
page_size: int = 100,
|
|
45
|
+
max_pages: int = 25,
|
|
46
|
+
) -> int:
|
|
47
|
+
params: dict = {"filing_year": year, "page_size": page_size}
|
|
48
|
+
if client:
|
|
49
|
+
params["client_name"] = client
|
|
50
|
+
if registrant:
|
|
51
|
+
params["registrant_name"] = registrant
|
|
52
|
+
|
|
53
|
+
headers = {"Authorization": f"Token {token}"} if token else None
|
|
54
|
+
rows: list[dict[str, str]] = []
|
|
55
|
+
url = ENDPOINT
|
|
56
|
+
page = 0
|
|
57
|
+
while page < max_pages:
|
|
58
|
+
try:
|
|
59
|
+
payload = get_json(url, params=params if page == 0 else None, headers=headers)
|
|
60
|
+
except Exception as e: # noqa: BLE001
|
|
61
|
+
print(f"Senate LDA error on page {page + 1}: {e}", file=sys.stderr)
|
|
62
|
+
break
|
|
63
|
+
if not isinstance(payload, dict):
|
|
64
|
+
break
|
|
65
|
+
results = payload.get("results", [])
|
|
66
|
+
for r in results:
|
|
67
|
+
client_obj = r.get("client") or {}
|
|
68
|
+
registrant_obj = r.get("registrant") or {}
|
|
69
|
+
lobbying_activities = r.get("lobbying_activities") or []
|
|
70
|
+
lobbyists = []
|
|
71
|
+
issues = []
|
|
72
|
+
entities = []
|
|
73
|
+
for la in lobbying_activities:
|
|
74
|
+
for lob in la.get("lobbyists") or []:
|
|
75
|
+
lob_obj = lob.get("lobbyist") or {}
|
|
76
|
+
name = " ".join(
|
|
77
|
+
x for x in (lob_obj.get("first_name", ""), lob_obj.get("last_name", "")) if x
|
|
78
|
+
)
|
|
79
|
+
if name:
|
|
80
|
+
lobbyists.append(name)
|
|
81
|
+
desc = la.get("description") or ""
|
|
82
|
+
if desc:
|
|
83
|
+
issues.append(desc)
|
|
84
|
+
for ge in la.get("government_entities") or []:
|
|
85
|
+
nm = ge.get("name") or ""
|
|
86
|
+
if nm:
|
|
87
|
+
entities.append(nm)
|
|
88
|
+
rows.append(
|
|
89
|
+
{
|
|
90
|
+
"filing_uuid": r.get("filing_uuid", "") or "",
|
|
91
|
+
"filing_type": r.get("filing_type", "") or "",
|
|
92
|
+
"filing_year": str(r.get("filing_year", "") or year),
|
|
93
|
+
"filing_period": r.get("filing_period", "") or "",
|
|
94
|
+
"registrant_name": registrant_obj.get("name", "") or "",
|
|
95
|
+
"registrant_id": str(registrant_obj.get("id", "") or ""),
|
|
96
|
+
"client_name": client_obj.get("name", "") or "",
|
|
97
|
+
"client_id": str(client_obj.get("id", "") or ""),
|
|
98
|
+
"client_general_description": client_obj.get("general_description", "") or "",
|
|
99
|
+
"income": str(r.get("income", "") or ""),
|
|
100
|
+
"expenses": str(r.get("expenses", "") or ""),
|
|
101
|
+
"lobbyists": "; ".join(sorted(set(lobbyists))),
|
|
102
|
+
"issues": "; ".join(issues),
|
|
103
|
+
"government_entities": "; ".join(sorted(set(entities))),
|
|
104
|
+
"filing_date": (r.get("dt_posted") or "")[:10],
|
|
105
|
+
}
|
|
106
|
+
)
|
|
107
|
+
next_url = payload.get("next")
|
|
108
|
+
if not next_url:
|
|
109
|
+
break
|
|
110
|
+
url = next_url
|
|
111
|
+
page += 1
|
|
112
|
+
time.sleep(1.0 if not token else 0.3)
|
|
113
|
+
|
|
114
|
+
Path(out_path).parent.mkdir(parents=True, exist_ok=True)
|
|
115
|
+
with open(out_path, "w", newline="", encoding="utf-8") as fh:
|
|
116
|
+
w = csv.DictWriter(fh, fieldnames=COLUMNS)
|
|
117
|
+
w.writeheader()
|
|
118
|
+
w.writerows(rows)
|
|
119
|
+
return len(rows)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def main() -> int:
|
|
123
|
+
p = argparse.ArgumentParser(description=__doc__)
|
|
124
|
+
p.add_argument("--client", help="Client name filter")
|
|
125
|
+
p.add_argument("--registrant", help="Registrant (lobbying firm) name filter")
|
|
126
|
+
p.add_argument("--year", type=int, default=2024)
|
|
127
|
+
p.add_argument("--token", default=os.environ.get("SENATE_LDA_TOKEN"))
|
|
128
|
+
p.add_argument("--max-pages", type=int, default=25)
|
|
129
|
+
p.add_argument("--out", required=True)
|
|
130
|
+
a = p.parse_args()
|
|
131
|
+
if not (a.client or a.registrant):
|
|
132
|
+
p.error("must supply at least one of --client / --registrant")
|
|
133
|
+
n = fetch(
|
|
134
|
+
client=a.client,
|
|
135
|
+
registrant=a.registrant,
|
|
136
|
+
year=a.year,
|
|
137
|
+
token=a.token,
|
|
138
|
+
out_path=a.out,
|
|
139
|
+
max_pages=a.max_pages,
|
|
140
|
+
)
|
|
141
|
+
print(f"Wrote {n} Senate LDA rows to {a.out}")
|
|
142
|
+
return 0
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
if __name__ == "__main__":
|
|
146
|
+
raise SystemExit(main())
|