@egbe-ai/opencode 0.0.0-stage → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +124 -2
- package/index.js +5 -0
- package/package.json +55 -4
- package/plugin.js +230 -0
- package/session-log.js +315 -0
- package/skills/.egbe-catalog/manifest.json +1 -0
- package/skills/egbe-os-analytics/SKILL.md +499 -0
- package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
- package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
- package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
- package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
- package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
- package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
- package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
- package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
- package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
- package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
- package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
- package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
- package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
- package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
- package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
- package/skills/egbe-os-db/SKILL.md +174 -0
- package/skills/egbe-os-deploy/SKILL.md +412 -0
- package/skills/egbe-os-design/SKILL.md +182 -0
- package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
- package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
- package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
- package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
- package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
- package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
- package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
- package/skills/egbe-os-destination-readiness/references/report.md +116 -0
- package/skills/egbe-os-domains/SKILL.md +189 -0
- package/skills/egbe-os-errors/SKILL.md +259 -0
- package/skills/egbe-os-git/SKILL.md +275 -0
- package/skills/egbe-os-git/references/issues.md +33 -0
- package/skills/egbe-os-google-ads/SKILL.md +163 -0
- package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
- package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
- package/skills/egbe-os-google-ads/references/conversions.md +153 -0
- package/skills/egbe-os-google-ads/references/delivery.md +128 -0
- package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
- package/skills/egbe-os-google-ads/references/reporting.md +105 -0
- package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
- package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
- package/skills/egbe-os-google-auth/SKILL.md +285 -0
- package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
- package/skills/egbe-os-images/SKILL.md +163 -0
- package/skills/egbe-os-images/references/stock-images.md +21 -0
- package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
- package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
- package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
- package/skills/egbe-os-llm/SKILL.md +160 -0
- package/skills/egbe-os-mail/SKILL.md +180 -0
- package/skills/egbe-os-me/SKILL.md +119 -0
- package/skills/egbe-os-meta-ads/SKILL.md +114 -0
- package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
- package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
- package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
- package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
- package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
- package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
- package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
- package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
- package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
- package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
- package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
- package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
- package/skills/egbe-os-notifications/SKILL.md +78 -0
- package/skills/egbe-os-openai-ads/SKILL.md +149 -0
- package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
- package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
- package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
- package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
- package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
- package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
- package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
- package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
- package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
- package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
- package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
- package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
- package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
- package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
- package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
- package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
- package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
- package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
- package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
- package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
- package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
- package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
- package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
- package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
- package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
- package/skills/egbe-os-payments/SKILL.md +215 -0
- package/skills/egbe-os-payments/agents/openai.yaml +4 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
- package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
- package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
- package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
- package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
- package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
- package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
- package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
- package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
- package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
- package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
- package/skills/egbe-os-playbook/SKILL.md +79 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
- package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
- package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
- package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
- package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
- package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
- package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
- package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
- package/skills/egbe-os-playbook/playbooks/product.md +67 -0
- package/skills/egbe-os-playbook/references/catalog.md +127 -0
- package/skills/egbe-os-playbook/references/changes.md +167 -0
- package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
- package/skills/egbe-os-product-auth/SKILL.md +151 -0
- package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
- package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
- package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
- package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
- package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
- package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
- package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
- package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
- package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
- package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
- package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
- package/skills/egbe-os-product-build/SKILL.md +111 -0
- package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-build/references/capabilities.md +25 -0
- package/skills/egbe-os-product-build/references/engineering.md +72 -0
- package/skills/egbe-os-product-build/references/execution.md +55 -0
- package/skills/egbe-os-product-build/references/foundations.md +93 -0
- package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
- package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
- package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
- package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
- package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
- package/skills/egbe-os-product-market-research/SKILL.md +159 -0
- package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
- package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
- package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
- package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
- package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
- package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
- package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
- package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
- package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
- package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
- package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
- package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
- package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
- package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
- package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
- package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
- package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
- package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
- package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
- package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
- package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
- package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
- package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
- package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
- package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
- package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
- package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
- package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
- package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
- package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
- package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
- package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
- package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
- package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
- package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
- package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
- package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
- package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
- package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
- package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
- package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
- package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
- package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
- package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
- package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
- package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
- package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
- package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
- package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
- package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
- package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
- package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
- package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
- package/skills/egbe-os-product-report/SKILL.md +150 -0
- package/skills/egbe-os-product-report/references/artifacts.md +84 -0
- package/skills/egbe-os-product-report/references/brief.md +34 -0
- package/skills/egbe-os-product-report/references/daily.md +76 -0
- package/skills/egbe-os-product-report/references/sources.md +68 -0
- package/skills/egbe-os-product-routines/SKILL.md +166 -0
- package/skills/egbe-os-product-routines/references/catalog.md +22 -0
- package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
- package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
- package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
- package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
- package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
- package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
- package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
- package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
- package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
- package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
- package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
- package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
- package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
- package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
- package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
- package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
- package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
- package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
- package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
- package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
- package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
- package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
- package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
- package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
- package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
- package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
- package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
- package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
- package/skills/egbe-os-report-ui/SKILL.md +133 -0
- package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
- package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
- package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
- package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
- package/skills/egbe-os-report-ui/references/components.md +209 -0
- package/skills/egbe-os-research/SKILL.md +125 -0
- package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
- package/skills/egbe-os-scrape/SKILL.md +160 -0
- package/skills/egbe-os-search/SKILL.md +182 -0
- package/skills/egbe-os-search/references/google-places.md +81 -0
- package/skills/egbe-os-support/SKILL.md +149 -0
- package/skills/egbe-os-telemetry/SKILL.md +344 -0
- package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
- package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
- package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
- package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
- package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
- package/skills/egbe-os-tools/SKILL.md +180 -0
- package/skills/egbe-os-tools/references/theona.md +26 -0
- package/skills/egbe-os-wallet/SKILL.md +181 -0
- package/skills/egbe-setup/EGBE.md +484 -0
- package/skills/egbe-setup/SKILL.md +116 -0
- package/skills/egbe-setup/references/project.md +80 -0
- package/state.js +37 -0
- package/update.js +361 -0
|
@@ -0,0 +1,1133 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Shared deterministic contracts for the organic-trend-watch skill."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import sys as _artifact_sys
|
|
7
|
+
from pathlib import Path as _ArtifactPath
|
|
8
|
+
_artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
|
|
9
|
+
from egbe_artifacts import artifact_path as _artifact_path
|
|
10
|
+
|
|
11
|
+
import fcntl
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import re
|
|
16
|
+
import tempfile
|
|
17
|
+
import urllib.parse
|
|
18
|
+
import uuid
|
|
19
|
+
from contextlib import contextmanager
|
|
20
|
+
from datetime import datetime, timezone
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
SCHEMA_VERSION = 1
|
|
26
|
+
SECTION_ID = "organic_trend_watch"
|
|
27
|
+
SECTION_HEADING = "TikTok/Reels market trend research for this product"
|
|
28
|
+
ANALYSIS_CONTRACT_VERSION = "evidence-v5"
|
|
29
|
+
MULTIMODAL_ANALYSIS_CONTRACT = "organic-video-evidence.v1"
|
|
30
|
+
MEDIA_PROMPT_CONTRACT_VERSION = "media-evidence-v1"
|
|
31
|
+
VISUAL_ANALYSIS_PROMPT_PATH = (
|
|
32
|
+
Path(__file__).resolve().parent.parent / "references/visual-analysis-prompt.md"
|
|
33
|
+
)
|
|
34
|
+
PROVIDER_INPUT_CONTRACT_VERSION = "apify-inputs-v1"
|
|
35
|
+
TERMINAL_PROVIDER_FAILURE_STATUSES = {"FAILED", "ABORTED", "TIMED-OUT", "TIMED_OUT"}
|
|
36
|
+
RUNTIME_RELATIVE = Path(_artifact_path("egbe-docs/research/market/work/organic_trend_watch"))
|
|
37
|
+
STEPS = (
|
|
38
|
+
"discovery",
|
|
39
|
+
"collection",
|
|
40
|
+
"normalization",
|
|
41
|
+
"visual_analysis",
|
|
42
|
+
"synthesis",
|
|
43
|
+
"render",
|
|
44
|
+
)
|
|
45
|
+
CANONICAL_PROVIDER_RUNS = {
|
|
46
|
+
"tiktok-discovery": {
|
|
47
|
+
"platform": "tiktok",
|
|
48
|
+
"kind": "discovery",
|
|
49
|
+
"actor_id": "clockworks/tiktok-scraper",
|
|
50
|
+
"input_keys": (
|
|
51
|
+
"oldestPostDateUnified",
|
|
52
|
+
"profileSorting",
|
|
53
|
+
"profiles",
|
|
54
|
+
"resultsPerPage",
|
|
55
|
+
),
|
|
56
|
+
},
|
|
57
|
+
"instagram-discovery": {
|
|
58
|
+
"platform": "instagram",
|
|
59
|
+
"kind": "discovery",
|
|
60
|
+
"actor_id": "apify/instagram-reel-scraper",
|
|
61
|
+
"input_keys": ("onlyPostsNewerThan", "resultsLimit", "username"),
|
|
62
|
+
},
|
|
63
|
+
"tiktok-enrichment": {
|
|
64
|
+
"platform": "tiktok",
|
|
65
|
+
"kind": "enrichment",
|
|
66
|
+
"actor_id": "clockworks/tiktok-scraper",
|
|
67
|
+
"input_keys": (
|
|
68
|
+
"commentsPerPost",
|
|
69
|
+
"downloadSubtitlesOptions",
|
|
70
|
+
"postURLs",
|
|
71
|
+
"shouldDownloadVideos",
|
|
72
|
+
),
|
|
73
|
+
},
|
|
74
|
+
"instagram-enrichment": {
|
|
75
|
+
"platform": "instagram",
|
|
76
|
+
"kind": "enrichment",
|
|
77
|
+
"actor_id": "apify/instagram-reel-scraper",
|
|
78
|
+
"input_keys": (
|
|
79
|
+
"includeDownloadedVideo",
|
|
80
|
+
"includeSharesCount",
|
|
81
|
+
"includeTranscript",
|
|
82
|
+
"username",
|
|
83
|
+
),
|
|
84
|
+
},
|
|
85
|
+
}
|
|
86
|
+
TIKTOK_COMMENTS_BUNDLE_PATH = "raw/tiktok-enrichment-comments.json"
|
|
87
|
+
TIKTOK_COMMENTS_LINK_KIND = "tiktok-comments"
|
|
88
|
+
VISUAL_EVIDENCE_METHOD = "provider-media-v2"
|
|
89
|
+
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
90
|
+
APIFY_DATASET_ID_RE = re.compile(r"^[A-Za-z0-9_-]{5,128}$")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class ContractError(ValueError):
|
|
94
|
+
"""Raised when a runtime artifact violates the public contract."""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@contextmanager
|
|
98
|
+
def provider_run_lock(root: Path):
|
|
99
|
+
"""Serialize provider-input rebuilding with submission and recovery."""
|
|
100
|
+
descriptor = os.open(root / ".apify-run.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
|
|
101
|
+
try:
|
|
102
|
+
try:
|
|
103
|
+
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
104
|
+
except BlockingIOError as exc:
|
|
105
|
+
raise ContractError("another provider operation holds this runtime; retry after it finishes") from exc
|
|
106
|
+
yield
|
|
107
|
+
finally:
|
|
108
|
+
# Keep the inode in place; replacing it could let a concurrent caller
|
|
109
|
+
# acquire a different lock for the same runtime.
|
|
110
|
+
os.close(descriptor)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def sha256_bytes(value: bytes) -> str:
|
|
114
|
+
return hashlib.sha256(value).hexdigest()
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def sha256_file(path: Path) -> str:
|
|
118
|
+
digest = hashlib.sha256()
|
|
119
|
+
with path.open("rb") as handle:
|
|
120
|
+
for chunk in iter(lambda: handle.read(64 * 1024), b""):
|
|
121
|
+
digest.update(chunk)
|
|
122
|
+
return digest.hexdigest()
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def load_visual_analysis_instructions() -> str:
|
|
126
|
+
try:
|
|
127
|
+
value = VISUAL_ANALYSIS_PROMPT_PATH.read_text(encoding="utf-8").strip()
|
|
128
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
129
|
+
raise ContractError(f"cannot read visual analysis prompt: {exc}") from exc
|
|
130
|
+
if not value:
|
|
131
|
+
raise ContractError("visual analysis prompt is empty")
|
|
132
|
+
return value
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def canonical_frame_seconds(duration: Any) -> tuple[float, ...]:
|
|
136
|
+
opening = [0.0, 0.5, 1.5, 3.0, 5.0]
|
|
137
|
+
if not isinstance(duration, (int, float)) or isinstance(duration, bool) or duration < 0:
|
|
138
|
+
return tuple(opening)
|
|
139
|
+
seconds = [value for value in opening if value <= float(duration)]
|
|
140
|
+
if duration > 6:
|
|
141
|
+
seconds.extend(
|
|
142
|
+
[
|
|
143
|
+
round(float(duration) * 0.5, 3),
|
|
144
|
+
round(float(duration) * 0.8, 3),
|
|
145
|
+
round(max(0.0, float(duration) - 0.5), 3),
|
|
146
|
+
]
|
|
147
|
+
)
|
|
148
|
+
return tuple(sorted(set(seconds)))
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def visual_analysis_receipt_path(root: Path, observation_id: str) -> Path:
|
|
152
|
+
safe = "".join(
|
|
153
|
+
character if character.isalnum() else "-" for character in observation_id
|
|
154
|
+
).strip("-") or "observation"
|
|
155
|
+
suffix = sha256_bytes(observation_id.encode("utf-8"))[:12]
|
|
156
|
+
return root / "visual-analysis" / f"{safe}-{suffix}.json"
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def multimodal_input_sha256(
|
|
160
|
+
*,
|
|
161
|
+
analysis_contract: str,
|
|
162
|
+
instructions: str,
|
|
163
|
+
frames: list[dict[str, Any]],
|
|
164
|
+
transcript: str | None,
|
|
165
|
+
) -> str:
|
|
166
|
+
value = {
|
|
167
|
+
"schema_version": SCHEMA_VERSION,
|
|
168
|
+
"analysis_contract": analysis_contract,
|
|
169
|
+
"instructions": instructions,
|
|
170
|
+
"frames": [
|
|
171
|
+
{
|
|
172
|
+
"ref": frame["ref"],
|
|
173
|
+
"timestamp_seconds": frame["timestamp_seconds"],
|
|
174
|
+
"media_type": frame["media_type"],
|
|
175
|
+
"sha256": frame["sha256"],
|
|
176
|
+
}
|
|
177
|
+
for frame in frames
|
|
178
|
+
],
|
|
179
|
+
"transcript_sha256": (
|
|
180
|
+
sha256_bytes(transcript.encode("utf-8")) if transcript else None
|
|
181
|
+
),
|
|
182
|
+
}
|
|
183
|
+
return sha256_bytes(
|
|
184
|
+
json.dumps(
|
|
185
|
+
value,
|
|
186
|
+
sort_keys=True,
|
|
187
|
+
separators=(",", ":"),
|
|
188
|
+
ensure_ascii=False,
|
|
189
|
+
).encode("utf-8")
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def read_json(path: Path) -> dict[str, Any]:
|
|
194
|
+
try:
|
|
195
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
196
|
+
except FileNotFoundError as exc:
|
|
197
|
+
raise ContractError(f"missing JSON artifact: {path}") from exc
|
|
198
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
199
|
+
raise ContractError(f"invalid JSON artifact {path}: {exc}") from exc
|
|
200
|
+
if not isinstance(value, dict):
|
|
201
|
+
raise ContractError(f"JSON artifact must contain an object: {path}")
|
|
202
|
+
return value
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def atomic_write_json(path: Path, payload: dict[str, Any]) -> None:
|
|
206
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
207
|
+
temporary: Path | None = None
|
|
208
|
+
try:
|
|
209
|
+
with tempfile.NamedTemporaryFile(
|
|
210
|
+
mode="w",
|
|
211
|
+
encoding="utf-8",
|
|
212
|
+
newline="\n",
|
|
213
|
+
dir=path.parent,
|
|
214
|
+
prefix=f".{path.name}.",
|
|
215
|
+
suffix=".tmp",
|
|
216
|
+
delete=False,
|
|
217
|
+
) as handle:
|
|
218
|
+
json.dump(payload, handle, indent=2, sort_keys=True, ensure_ascii=False)
|
|
219
|
+
handle.write("\n")
|
|
220
|
+
handle.flush()
|
|
221
|
+
os.fsync(handle.fileno())
|
|
222
|
+
temporary = Path(handle.name)
|
|
223
|
+
os.chmod(temporary, 0o600)
|
|
224
|
+
os.replace(temporary, path)
|
|
225
|
+
temporary = None
|
|
226
|
+
finally:
|
|
227
|
+
if temporary is not None:
|
|
228
|
+
temporary.unlink(missing_ok=True)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def atomic_write_text(path: Path, value: str) -> None:
|
|
232
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
233
|
+
temporary: Path | None = None
|
|
234
|
+
try:
|
|
235
|
+
with tempfile.NamedTemporaryFile(
|
|
236
|
+
mode="w",
|
|
237
|
+
encoding="utf-8",
|
|
238
|
+
newline="\n",
|
|
239
|
+
dir=path.parent,
|
|
240
|
+
prefix=f".{path.name}.",
|
|
241
|
+
suffix=".tmp",
|
|
242
|
+
delete=False,
|
|
243
|
+
) as handle:
|
|
244
|
+
handle.write(value)
|
|
245
|
+
handle.flush()
|
|
246
|
+
os.fsync(handle.fileno())
|
|
247
|
+
temporary = Path(handle.name)
|
|
248
|
+
os.chmod(temporary, 0o600)
|
|
249
|
+
os.replace(temporary, path)
|
|
250
|
+
temporary = None
|
|
251
|
+
finally:
|
|
252
|
+
if temporary is not None:
|
|
253
|
+
temporary.unlink(missing_ok=True)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def require_string(value: Any, field: str, *, allow_empty: bool = False) -> str:
|
|
257
|
+
if not isinstance(value, str) or (not allow_empty and not value.strip()):
|
|
258
|
+
raise ContractError(f"{field} must be a non-empty string")
|
|
259
|
+
return value
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def require_sha256(value: Any, field: str) -> str:
|
|
263
|
+
if not isinstance(value, str) or not SHA256_RE.fullmatch(value):
|
|
264
|
+
raise ContractError(f"{field} must be a lowercase SHA-256 hex digest")
|
|
265
|
+
return value
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def validate_visual_analysis_receipt(
|
|
269
|
+
root: Path,
|
|
270
|
+
observation: dict[str, Any],
|
|
271
|
+
*,
|
|
272
|
+
instructions: str,
|
|
273
|
+
) -> dict[str, Any]:
|
|
274
|
+
observation_id = require_string(
|
|
275
|
+
observation.get("observation_id"), "normalized observation_id"
|
|
276
|
+
)
|
|
277
|
+
path = visual_analysis_receipt_path(root, observation_id)
|
|
278
|
+
value = read_json(path)
|
|
279
|
+
if value.get("schema_version") != SCHEMA_VERSION:
|
|
280
|
+
raise ContractError(f"{path.name} schema_version must be 1")
|
|
281
|
+
if value.get("observation_id") != observation_id:
|
|
282
|
+
raise ContractError(f"{path.name} belongs to another observation")
|
|
283
|
+
if value.get("analysis_contract") != MULTIMODAL_ANALYSIS_CONTRACT:
|
|
284
|
+
raise ContractError(f"{path.name} analysis contract mismatch")
|
|
285
|
+
|
|
286
|
+
evidence = observation.get("evidence")
|
|
287
|
+
if not isinstance(evidence, dict):
|
|
288
|
+
raise ContractError(f"{observation_id} has no evidence object")
|
|
289
|
+
preparation = evidence.get("visual_preparation")
|
|
290
|
+
samples = preparation.get("frame_samples") if isinstance(preparation, dict) else []
|
|
291
|
+
if not isinstance(samples, list):
|
|
292
|
+
raise ContractError(f"{observation_id} frame samples are invalid")
|
|
293
|
+
hashes = evidence.get("artifact_sha256")
|
|
294
|
+
if not isinstance(hashes, dict):
|
|
295
|
+
raise ContractError(f"{observation_id} artifact hashes are invalid")
|
|
296
|
+
frames: list[dict[str, Any]] = []
|
|
297
|
+
for index, sample in enumerate(samples):
|
|
298
|
+
if not isinstance(sample, dict):
|
|
299
|
+
raise ContractError(f"{observation_id} frame sample {index} is invalid")
|
|
300
|
+
ref = require_string(sample.get("path"), f"{observation_id} frame ref")
|
|
301
|
+
timestamp = sample.get("second")
|
|
302
|
+
if not isinstance(timestamp, (int, float)) or isinstance(timestamp, bool):
|
|
303
|
+
raise ContractError(f"{observation_id} frame timestamp is invalid")
|
|
304
|
+
frame_path = resolve_relative(root, ref, f"{observation_id} frame ref")
|
|
305
|
+
digest = require_sha256(hashes.get(ref), f"{observation_id} frame sha256")
|
|
306
|
+
if not frame_path.is_file() or sha256_file(frame_path) != digest:
|
|
307
|
+
raise ContractError(f"{observation_id} frame hash mismatch: {ref}")
|
|
308
|
+
media_type = {
|
|
309
|
+
".jpg": "image/jpeg",
|
|
310
|
+
".jpeg": "image/jpeg",
|
|
311
|
+
".png": "image/png",
|
|
312
|
+
".webp": "image/webp",
|
|
313
|
+
}.get(frame_path.suffix.casefold())
|
|
314
|
+
if media_type is None:
|
|
315
|
+
raise ContractError(f"{observation_id} frame media type is unsupported")
|
|
316
|
+
frames.append(
|
|
317
|
+
{
|
|
318
|
+
"ref": ref,
|
|
319
|
+
"timestamp_seconds": float(timestamp),
|
|
320
|
+
"media_type": media_type,
|
|
321
|
+
"sha256": digest,
|
|
322
|
+
}
|
|
323
|
+
)
|
|
324
|
+
transcript_value = evidence.get("transcript")
|
|
325
|
+
transcript = (
|
|
326
|
+
transcript_value.strip()
|
|
327
|
+
if isinstance(transcript_value, str) and transcript_value.strip()
|
|
328
|
+
else None
|
|
329
|
+
)
|
|
330
|
+
expected_input = multimodal_input_sha256(
|
|
331
|
+
analysis_contract=MULTIMODAL_ANALYSIS_CONTRACT,
|
|
332
|
+
instructions=instructions,
|
|
333
|
+
frames=frames,
|
|
334
|
+
transcript=transcript,
|
|
335
|
+
)
|
|
336
|
+
if value.get("input_sha256") != expected_input:
|
|
337
|
+
raise ContractError(f"{path.name} input hash is stale")
|
|
338
|
+
if value.get("frames") != frames:
|
|
339
|
+
raise ContractError(f"{path.name} frame provenance mismatch")
|
|
340
|
+
expected_transcript = {
|
|
341
|
+
"status": "available" if transcript else "unavailable",
|
|
342
|
+
"sha256": sha256_bytes(transcript.encode("utf-8")) if transcript else None,
|
|
343
|
+
}
|
|
344
|
+
if value.get("transcript") != expected_transcript:
|
|
345
|
+
raise ContractError(f"{path.name} transcript provenance mismatch")
|
|
346
|
+
|
|
347
|
+
status = value.get("status")
|
|
348
|
+
if status not in {"completed", "unavailable"}:
|
|
349
|
+
raise ContractError(f"{path.name} status is invalid")
|
|
350
|
+
analysis = value.get("analysis")
|
|
351
|
+
if not isinstance(analysis, dict):
|
|
352
|
+
raise ContractError(f"{path.name} analysis must be an object")
|
|
353
|
+
expected_analysis_keys = {
|
|
354
|
+
"frames",
|
|
355
|
+
"sequence_summary",
|
|
356
|
+
"transcript",
|
|
357
|
+
"claims",
|
|
358
|
+
"gaps",
|
|
359
|
+
}
|
|
360
|
+
if set(analysis) != expected_analysis_keys:
|
|
361
|
+
raise ContractError(f"{path.name} analysis shape is invalid")
|
|
362
|
+
analysis_frames = analysis.get("frames")
|
|
363
|
+
if not isinstance(analysis_frames, list) or [
|
|
364
|
+
item.get("ref") if isinstance(item, dict) else None for item in analysis_frames
|
|
365
|
+
] != [frame["ref"] for frame in frames]:
|
|
366
|
+
raise ContractError(f"{path.name} analyzed frames mismatch")
|
|
367
|
+
for index, item in enumerate(analysis_frames):
|
|
368
|
+
if set(item) != {"ref", "visible_text", "observations"}:
|
|
369
|
+
raise ContractError(f"{path.name} frame analysis {index} shape is invalid")
|
|
370
|
+
for field in ("visible_text", "observations"):
|
|
371
|
+
values = item.get(field)
|
|
372
|
+
if not isinstance(values, list) or not all(
|
|
373
|
+
isinstance(entry, str) and entry.strip() for entry in values
|
|
374
|
+
):
|
|
375
|
+
raise ContractError(
|
|
376
|
+
f"{path.name} frame analysis {index}.{field} is invalid"
|
|
377
|
+
)
|
|
378
|
+
if not isinstance(analysis.get("sequence_summary"), str):
|
|
379
|
+
raise ContractError(f"{path.name} sequence_summary must be a string")
|
|
380
|
+
transcript_analysis = analysis.get("transcript")
|
|
381
|
+
if not isinstance(transcript_analysis, dict) or set(transcript_analysis) != {
|
|
382
|
+
"status",
|
|
383
|
+
"observations",
|
|
384
|
+
}:
|
|
385
|
+
raise ContractError(f"{path.name} transcript analysis shape is invalid")
|
|
386
|
+
if transcript_analysis.get("status") != expected_transcript["status"]:
|
|
387
|
+
raise ContractError(f"{path.name} transcript analysis status mismatch")
|
|
388
|
+
transcript_observations = transcript_analysis.get("observations")
|
|
389
|
+
if not isinstance(transcript_observations, list) or not all(
|
|
390
|
+
isinstance(entry, str) and entry.strip() for entry in transcript_observations
|
|
391
|
+
):
|
|
392
|
+
raise ContractError(f"{path.name} transcript observations are invalid")
|
|
393
|
+
if not transcript and transcript_observations:
|
|
394
|
+
raise ContractError(f"{path.name} invented observations for a missing transcript")
|
|
395
|
+
allowed_refs = {frame["ref"] for frame in frames}
|
|
396
|
+
if transcript:
|
|
397
|
+
allowed_refs.add("transcript")
|
|
398
|
+
claims = analysis.get("claims")
|
|
399
|
+
if not isinstance(claims, list):
|
|
400
|
+
raise ContractError(f"{path.name} claims must be a list")
|
|
401
|
+
for index, claim in enumerate(claims):
|
|
402
|
+
if not isinstance(claim, dict) or set(claim) != {
|
|
403
|
+
"text",
|
|
404
|
+
"modality",
|
|
405
|
+
"evidence_refs",
|
|
406
|
+
}:
|
|
407
|
+
raise ContractError(f"{path.name} claim {index} shape is invalid")
|
|
408
|
+
require_string(claim.get("text"), f"{path.name} claim {index}.text")
|
|
409
|
+
modality = claim.get("modality")
|
|
410
|
+
refs = claim.get("evidence_refs")
|
|
411
|
+
if modality not in {"visual", "verbal", "mixed"}:
|
|
412
|
+
raise ContractError(f"{path.name} claim {index} modality is invalid")
|
|
413
|
+
if not isinstance(refs, list) or not refs or len(refs) != len(set(refs)):
|
|
414
|
+
raise ContractError(f"{path.name} claim {index} refs are invalid")
|
|
415
|
+
if any(ref not in allowed_refs for ref in refs):
|
|
416
|
+
raise ContractError(f"{path.name} claim {index} cites unknown evidence")
|
|
417
|
+
has_visual = bool(set(refs) & {frame["ref"] for frame in frames})
|
|
418
|
+
has_verbal = "transcript" in refs
|
|
419
|
+
if modality == "visual" and (not has_visual or has_verbal):
|
|
420
|
+
raise ContractError(f"{path.name} visual claim refs are invalid")
|
|
421
|
+
if modality == "verbal" and (has_visual or not has_verbal):
|
|
422
|
+
raise ContractError(f"{path.name} verbal claim refs are invalid")
|
|
423
|
+
if modality == "mixed" and (not has_visual or not has_verbal):
|
|
424
|
+
raise ContractError(f"{path.name} mixed claim refs are invalid")
|
|
425
|
+
gaps = analysis.get("gaps")
|
|
426
|
+
if not isinstance(gaps, list) or not all(
|
|
427
|
+
isinstance(gap, str) and gap.strip() for gap in gaps
|
|
428
|
+
):
|
|
429
|
+
raise ContractError(f"{path.name} gaps are invalid")
|
|
430
|
+
|
|
431
|
+
if status == "unavailable":
|
|
432
|
+
if frames or transcript or analysis_frames or claims:
|
|
433
|
+
raise ContractError(f"{path.name} unavailable receipt contains evidence")
|
|
434
|
+
if value.get("model_role") is not None or value.get("effective_model") is not None:
|
|
435
|
+
raise ContractError(f"{path.name} unavailable receipt cannot name a model")
|
|
436
|
+
if value.get("prompt_contract_version") is not None:
|
|
437
|
+
raise ContractError(f"{path.name} unavailable receipt cannot name a prompt")
|
|
438
|
+
else:
|
|
439
|
+
if not frames and not transcript:
|
|
440
|
+
raise ContractError(f"{path.name} completed receipt has no evidence")
|
|
441
|
+
if value.get("prompt_contract_version") != MEDIA_PROMPT_CONTRACT_VERSION:
|
|
442
|
+
raise ContractError(f"{path.name} media prompt contract mismatch")
|
|
443
|
+
require_string(value.get("model_role"), f"{path.name} model_role")
|
|
444
|
+
require_string(value.get("effective_model"), f"{path.name} effective_model")
|
|
445
|
+
parse_rfc3339(value.get("completed_at"), f"{path.name} completed_at")
|
|
446
|
+
return value
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def parse_rfc3339(value: Any, field: str) -> datetime:
|
|
450
|
+
text = require_string(value, field)
|
|
451
|
+
normalized = text[:-1] + "+00:00" if text.endswith("Z") else text
|
|
452
|
+
try:
|
|
453
|
+
result = datetime.fromisoformat(normalized)
|
|
454
|
+
except ValueError as exc:
|
|
455
|
+
raise ContractError(f"{field} must be RFC3339: {text}") from exc
|
|
456
|
+
if result.tzinfo is None:
|
|
457
|
+
raise ContractError(f"{field} must include a timezone")
|
|
458
|
+
return result.astimezone(timezone.utc)
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def utc_now() -> str:
|
|
462
|
+
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace(
|
|
463
|
+
"+00:00", "Z"
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def resolve_relative(root: Path, value: Any, field: str) -> Path:
|
|
468
|
+
text = require_string(value, field)
|
|
469
|
+
relative = Path(text)
|
|
470
|
+
if relative.is_absolute() or ".." in relative.parts or relative == Path("."):
|
|
471
|
+
raise ContractError(f"{field} must be a safe relative path")
|
|
472
|
+
root_resolved = root.resolve()
|
|
473
|
+
candidate = (root / relative).resolve()
|
|
474
|
+
try:
|
|
475
|
+
candidate.relative_to(root_resolved)
|
|
476
|
+
except ValueError as exc:
|
|
477
|
+
raise ContractError(f"{field} escapes runtime root") from exc
|
|
478
|
+
return candidate
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def validate_context(path: Path) -> dict[str, Any]:
|
|
482
|
+
value = read_json(path)
|
|
483
|
+
if value.get("schema_version") != SCHEMA_VERSION:
|
|
484
|
+
raise ContractError("context.schema_version must be 1")
|
|
485
|
+
if value.get("section_id") != SECTION_ID:
|
|
486
|
+
raise ContractError(f"context.section_id must be {SECTION_ID}")
|
|
487
|
+
require_string(value.get("assignment"), "context.assignment")
|
|
488
|
+
require_string(value.get("product_context"), "context.product_context")
|
|
489
|
+
collection_parameters(value)
|
|
490
|
+
limit = value.get("max_analysis_videos")
|
|
491
|
+
if limit is not None and (isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 30):
|
|
492
|
+
raise ContractError("context.max_analysis_videos must be an integer from 1 to 30 (adapter resource ceiling)")
|
|
493
|
+
|
|
494
|
+
supporting = value.get("supporting_sections", [])
|
|
495
|
+
if not isinstance(supporting, list):
|
|
496
|
+
raise ContractError("context.supporting_sections must be a list")
|
|
497
|
+
ids: list[str] = []
|
|
498
|
+
for index, item in enumerate(supporting):
|
|
499
|
+
if not isinstance(item, dict):
|
|
500
|
+
raise ContractError(f"context.supporting_sections[{index}] must be an object")
|
|
501
|
+
section_id = require_string(
|
|
502
|
+
item.get("id"), f"context.supporting_sections[{index}].id"
|
|
503
|
+
)
|
|
504
|
+
require_string(item.get("label"), f"context.supporting_sections[{index}].label")
|
|
505
|
+
require_string(
|
|
506
|
+
item.get("content"), f"context.supporting_sections[{index}].content"
|
|
507
|
+
)
|
|
508
|
+
if section_id in ids:
|
|
509
|
+
raise ContractError(f"duplicate supporting section: {section_id}")
|
|
510
|
+
ids.append(section_id)
|
|
511
|
+
sources = value.get("sources", {})
|
|
512
|
+
if not isinstance(sources, dict):
|
|
513
|
+
raise ContractError("context.sources must be an object")
|
|
514
|
+
context_source = sources.get("context")
|
|
515
|
+
if context_source is not None:
|
|
516
|
+
if not isinstance(context_source, dict):
|
|
517
|
+
raise ContractError("context.sources.context must be an object")
|
|
518
|
+
require_string(context_source.get("path"), "context.sources.context.path")
|
|
519
|
+
require_sha256(context_source.get("source_sha256"), "context.sources.context.source_sha256")
|
|
520
|
+
source_sections = sources.get("supporting_sections", [])
|
|
521
|
+
if not isinstance(source_sections, list):
|
|
522
|
+
raise ContractError("context.sources.supporting_sections must be a list")
|
|
523
|
+
source_ids: list[str] = []
|
|
524
|
+
for index, item in enumerate(source_sections):
|
|
525
|
+
if not isinstance(item, dict):
|
|
526
|
+
raise ContractError(
|
|
527
|
+
f"context.sources.supporting_sections[{index}] must be an object"
|
|
528
|
+
)
|
|
529
|
+
section_id = require_string(
|
|
530
|
+
item.get("id"), f"context.sources.supporting_sections[{index}].id"
|
|
531
|
+
)
|
|
532
|
+
require_string(
|
|
533
|
+
item.get("path"), f"context.sources.supporting_sections[{index}].path"
|
|
534
|
+
)
|
|
535
|
+
require_sha256(
|
|
536
|
+
item.get("source_sha256"),
|
|
537
|
+
f"context.sources.supporting_sections[{index}].source_sha256",
|
|
538
|
+
)
|
|
539
|
+
source_ids.append(section_id)
|
|
540
|
+
if source_ids and source_ids != ids:
|
|
541
|
+
raise ContractError(
|
|
542
|
+
"context supporting content and source metadata must name the same sections"
|
|
543
|
+
)
|
|
544
|
+
return value
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def collection_parameters(context: dict[str, Any]) -> dict[str, int]:
|
|
548
|
+
"""Resolve the one caller-owned collection scope; not an evidence threshold."""
|
|
549
|
+
supplied = context.get("collection", {})
|
|
550
|
+
defaults = {"discovery_limit_per_account": 30, "lookback_days": 60}
|
|
551
|
+
if not isinstance(supplied, dict) or set(supplied) - set(defaults):
|
|
552
|
+
raise ContractError("context.collection has unsupported fields")
|
|
553
|
+
result = {**defaults, **supplied}
|
|
554
|
+
for name, value in result.items():
|
|
555
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
|
556
|
+
raise ContractError(f"context.collection.{name} must be a positive integer")
|
|
557
|
+
if result["discovery_limit_per_account"] > 30:
|
|
558
|
+
raise ContractError("context.collection.discovery_limit_per_account must be from 1 to 30")
|
|
559
|
+
return result
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def validate_accounts(value: Any, field: str = "accounts") -> list[dict[str, Any]]:
|
|
563
|
+
if not isinstance(value, list) or not 1 <= len(value) <= 6:
|
|
564
|
+
raise ContractError(f"{field} must contain 1–6 accounts")
|
|
565
|
+
seen: set[tuple[str, str]] = set()
|
|
566
|
+
for index, account in enumerate(value):
|
|
567
|
+
item_field = f"{field}[{index}]"
|
|
568
|
+
if not isinstance(account, dict):
|
|
569
|
+
raise ContractError(f"{item_field} must be an object")
|
|
570
|
+
if set(account) - {
|
|
571
|
+
"platform",
|
|
572
|
+
"handle",
|
|
573
|
+
"account_type",
|
|
574
|
+
"product_keywords",
|
|
575
|
+
"followers",
|
|
576
|
+
}:
|
|
577
|
+
raise ContractError(f"{item_field} contains unsupported fields")
|
|
578
|
+
platform = require_string(account.get("platform"), f"{item_field}.platform")
|
|
579
|
+
if platform not in {"tiktok", "instagram"}:
|
|
580
|
+
raise ContractError(f"{item_field}.platform is unsupported")
|
|
581
|
+
handle = require_string(account.get("handle"), f"{item_field}.handle")
|
|
582
|
+
account_type = require_string(
|
|
583
|
+
account.get("account_type"), f"{item_field}.account_type"
|
|
584
|
+
)
|
|
585
|
+
if account_type not in {
|
|
586
|
+
"official",
|
|
587
|
+
"shadow",
|
|
588
|
+
"partner",
|
|
589
|
+
"shadow-or-partner",
|
|
590
|
+
"viral",
|
|
591
|
+
}:
|
|
592
|
+
raise ContractError(f"{item_field}.account_type is unsupported")
|
|
593
|
+
keywords = account.get("product_keywords")
|
|
594
|
+
if not isinstance(keywords, list) or not all(
|
|
595
|
+
isinstance(item, str) and item.strip() for item in keywords
|
|
596
|
+
):
|
|
597
|
+
raise ContractError(f"{item_field}.product_keywords must be strings")
|
|
598
|
+
followers = account.get("followers")
|
|
599
|
+
if followers is not None and (
|
|
600
|
+
not isinstance(followers, (int, float))
|
|
601
|
+
or isinstance(followers, bool)
|
|
602
|
+
or followers < 0
|
|
603
|
+
):
|
|
604
|
+
raise ContractError(f"{item_field}.followers must be non-negative or null")
|
|
605
|
+
identity = (platform, handle.casefold().lstrip("@"))
|
|
606
|
+
if identity in seen:
|
|
607
|
+
raise ContractError(f"duplicate account: {platform}/{handle}")
|
|
608
|
+
seen.add(identity)
|
|
609
|
+
return value
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def validate_accounts_file(root: Path) -> tuple[list[dict[str, Any]], dict[str, str]]:
|
|
613
|
+
relative = "raw/accounts.json"
|
|
614
|
+
path = resolve_relative(root, relative, "accounts source")
|
|
615
|
+
value = read_json(path)
|
|
616
|
+
if set(value) != {"accounts"}:
|
|
617
|
+
raise ContractError("raw/accounts.json must contain exactly an accounts field")
|
|
618
|
+
accounts = validate_accounts(value.get("accounts"), "raw accounts")
|
|
619
|
+
return accounts, {"path": relative, "sha256": sha256_file(path)}
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def runtime_root_from_workspace(workspace: Path) -> Path:
|
|
623
|
+
return workspace.resolve() / RUNTIME_RELATIVE
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
def artifact_ref(root: Path, relative: str) -> dict[str, str]:
|
|
627
|
+
path = resolve_relative(root, relative, relative)
|
|
628
|
+
if not path.is_file():
|
|
629
|
+
raise ContractError(f"missing runtime artifact: {relative}")
|
|
630
|
+
return {"path": relative, "sha256": sha256_file(path)}
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def extract_rows(value: Any) -> list[dict[str, Any]]:
|
|
634
|
+
"""Recognize an empty dataset, but do not turn invalid responses into no-data."""
|
|
635
|
+
rows: Any = None
|
|
636
|
+
if isinstance(value, list):
|
|
637
|
+
rows = value
|
|
638
|
+
elif isinstance(value, dict):
|
|
639
|
+
data = value.get("data")
|
|
640
|
+
if isinstance(data, dict) and "rows" in data:
|
|
641
|
+
rows = data["rows"]
|
|
642
|
+
elif "rows" in value:
|
|
643
|
+
rows = value["rows"]
|
|
644
|
+
elif "items" in value:
|
|
645
|
+
rows = value["items"]
|
|
646
|
+
elif isinstance(data, list):
|
|
647
|
+
rows = data
|
|
648
|
+
if not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows):
|
|
649
|
+
raise ContractError("provider dataset must contain an explicit list of row objects")
|
|
650
|
+
return rows
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
def _nested_value(value: dict[str, Any], *paths: str) -> Any:
|
|
654
|
+
for path in paths:
|
|
655
|
+
current: Any = value
|
|
656
|
+
for part in path.split("."):
|
|
657
|
+
if not isinstance(current, dict) or part not in current:
|
|
658
|
+
current = None
|
|
659
|
+
break
|
|
660
|
+
current = current[part]
|
|
661
|
+
if current is not None:
|
|
662
|
+
return current
|
|
663
|
+
return None
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def _optional_text(value: Any) -> str | None:
|
|
667
|
+
if value is None:
|
|
668
|
+
return None
|
|
669
|
+
text = str(value).strip()
|
|
670
|
+
return text or None
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def apify_dataset_id_from_url(value: Any) -> str:
|
|
674
|
+
text = require_string(value, "commentsDatasetURL")
|
|
675
|
+
parsed = urllib.parse.urlparse(text)
|
|
676
|
+
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
|
677
|
+
raise ContractError("commentsDatasetURL must be an HTTP(S) URL")
|
|
678
|
+
parts = [urllib.parse.unquote(part) for part in parsed.path.split("/") if part]
|
|
679
|
+
try:
|
|
680
|
+
dataset_index = parts.index("datasets")
|
|
681
|
+
dataset_id = parts[dataset_index + 1]
|
|
682
|
+
except (ValueError, IndexError) as exc:
|
|
683
|
+
raise ContractError("commentsDatasetURL does not identify an Apify dataset") from exc
|
|
684
|
+
if APIFY_DATASET_ID_RE.fullmatch(dataset_id) is None:
|
|
685
|
+
raise ContractError("commentsDatasetURL contains an invalid Apify dataset ID")
|
|
686
|
+
return dataset_id
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
def linked_tiktok_comment_requests(
|
|
690
|
+
main_payload: Any,
|
|
691
|
+
) -> list[dict[str, Any]]:
|
|
692
|
+
"""Return the exact linked comment datasets declared by TikTok video rows."""
|
|
693
|
+
|
|
694
|
+
grouped: dict[str, dict[str, Any]] = {}
|
|
695
|
+
for row_index, row in enumerate(extract_rows(main_payload)):
|
|
696
|
+
dataset_url = _nested_value(row, "commentsDatasetURL", "commentsDatasetUrl")
|
|
697
|
+
if dataset_url is None:
|
|
698
|
+
continue
|
|
699
|
+
dataset_id = apify_dataset_id_from_url(dataset_url)
|
|
700
|
+
video_id = _optional_text(
|
|
701
|
+
_nested_value(row, "id", "videoId", "aweme_id")
|
|
702
|
+
)
|
|
703
|
+
if video_id is None:
|
|
704
|
+
raise ContractError(
|
|
705
|
+
f"TikTok row {row_index} has commentsDatasetURL but no video identity"
|
|
706
|
+
)
|
|
707
|
+
video_url = _optional_text(
|
|
708
|
+
_nested_value(row, "webVideoUrl", "postUrl", "inputUrl", "url")
|
|
709
|
+
)
|
|
710
|
+
parent = {"video_id": video_id, "video_url": video_url}
|
|
711
|
+
entry = grouped.setdefault(
|
|
712
|
+
dataset_id,
|
|
713
|
+
{
|
|
714
|
+
"dataset_id": dataset_id,
|
|
715
|
+
"source_urls": [],
|
|
716
|
+
"parents": [],
|
|
717
|
+
},
|
|
718
|
+
)
|
|
719
|
+
source_url = str(dataset_url).strip()
|
|
720
|
+
if source_url not in entry["source_urls"]:
|
|
721
|
+
entry["source_urls"].append(source_url)
|
|
722
|
+
if parent not in entry["parents"]:
|
|
723
|
+
entry["parents"].append(parent)
|
|
724
|
+
for entry in grouped.values():
|
|
725
|
+
entry["source_urls"].sort()
|
|
726
|
+
entry["parents"].sort(
|
|
727
|
+
key=lambda parent: (parent["video_id"], parent.get("video_url") or "")
|
|
728
|
+
)
|
|
729
|
+
return [grouped[dataset_id] for dataset_id in sorted(grouped)]
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
def validate_linked_tiktok_comments_bundle(
|
|
733
|
+
main_payload: Any,
|
|
734
|
+
bundle: Any,
|
|
735
|
+
) -> dict[str, list[dict[str, Any]]]:
|
|
736
|
+
"""Validate linked raw responses and group their comment rows by parent video."""
|
|
737
|
+
|
|
738
|
+
if not isinstance(bundle, dict) or bundle.get("schema_version") != SCHEMA_VERSION:
|
|
739
|
+
raise ContractError("TikTok linked comments bundle schema_version must be 1")
|
|
740
|
+
if bundle.get("source_logical_key") != "tiktok-enrichment":
|
|
741
|
+
raise ContractError("TikTok linked comments bundle has the wrong source run")
|
|
742
|
+
expected_source_hash = sha256_bytes(
|
|
743
|
+
(json.dumps(main_payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n").encode(
|
|
744
|
+
"utf-8"
|
|
745
|
+
)
|
|
746
|
+
)
|
|
747
|
+
if bundle.get("source_dataset_sha256") != expected_source_hash:
|
|
748
|
+
raise ContractError("TikTok linked comments source dataset hash mismatch")
|
|
749
|
+
expected = linked_tiktok_comment_requests(main_payload)
|
|
750
|
+
datasets = bundle.get("datasets")
|
|
751
|
+
if not isinstance(datasets, list) or len(datasets) != len(expected):
|
|
752
|
+
raise ContractError("TikTok linked comments datasets differ from actor output")
|
|
753
|
+
|
|
754
|
+
comments_by_video: dict[str, list[dict[str, Any]]] = {}
|
|
755
|
+
for index, (actual, expected_entry) in enumerate(zip(datasets, expected, strict=True)):
|
|
756
|
+
if not isinstance(actual, dict):
|
|
757
|
+
raise ContractError(f"TikTok linked comments dataset {index} must be an object")
|
|
758
|
+
for field in ("dataset_id", "source_urls", "parents"):
|
|
759
|
+
if actual.get(field) != expected_entry[field]:
|
|
760
|
+
raise ContractError(
|
|
761
|
+
f"TikTok linked comments dataset {index} changed {field}"
|
|
762
|
+
)
|
|
763
|
+
response = actual.get("response")
|
|
764
|
+
if not isinstance(response, dict):
|
|
765
|
+
raise ContractError(
|
|
766
|
+
f"TikTok linked comments dataset {index} has no raw OS response"
|
|
767
|
+
)
|
|
768
|
+
parents = expected_entry["parents"]
|
|
769
|
+
parent_ids = {parent["video_id"] for parent in parents}
|
|
770
|
+
parent_urls = {
|
|
771
|
+
parent["video_url"]: parent["video_id"]
|
|
772
|
+
for parent in parents
|
|
773
|
+
if parent.get("video_url")
|
|
774
|
+
}
|
|
775
|
+
for comment in extract_rows(response):
|
|
776
|
+
comment_video_id = _optional_text(
|
|
777
|
+
_nested_value(comment, "videoId", "aweme_id")
|
|
778
|
+
)
|
|
779
|
+
comment_video_url = _optional_text(
|
|
780
|
+
_nested_value(comment, "videoWebUrl", "webVideoUrl", "postUrl")
|
|
781
|
+
)
|
|
782
|
+
if comment_video_id not in parent_ids:
|
|
783
|
+
comment_video_id = parent_urls.get(comment_video_url or "")
|
|
784
|
+
if comment_video_id is None and len(parent_ids) == 1:
|
|
785
|
+
comment_video_id = next(iter(parent_ids))
|
|
786
|
+
if comment_video_id is None or comment_video_id not in parent_ids:
|
|
787
|
+
raise ContractError(
|
|
788
|
+
f"TikTok linked comment in dataset {actual['dataset_id']} "
|
|
789
|
+
"cannot be bound to a parent video"
|
|
790
|
+
)
|
|
791
|
+
comments_by_video.setdefault(comment_video_id, []).append(comment)
|
|
792
|
+
return comments_by_video
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
def validate_provider_run(value: Any, field: str) -> dict[str, Any]:
|
|
796
|
+
if not isinstance(value, dict):
|
|
797
|
+
raise ContractError(f"{field} must be an object")
|
|
798
|
+
result = {
|
|
799
|
+
"logical_key": require_string(
|
|
800
|
+
value.get("logical_key"), f"{field}.logical_key"
|
|
801
|
+
),
|
|
802
|
+
"provider": require_string(value.get("provider"), f"{field}.provider"),
|
|
803
|
+
"actor_id": require_string(value.get("actor_id"), f"{field}.actor_id"),
|
|
804
|
+
"run_id": require_string(value.get("run_id"), f"{field}.run_id"),
|
|
805
|
+
"dataset_id": require_string(value.get("dataset_id"), f"{field}.dataset_id"),
|
|
806
|
+
"status": require_string(value.get("status"), f"{field}.status"),
|
|
807
|
+
"input_sha256": require_sha256(
|
|
808
|
+
value.get("input_sha256"), f"{field}.input_sha256"
|
|
809
|
+
),
|
|
810
|
+
"input_contract_version": require_string(
|
|
811
|
+
value.get("input_contract_version"), f"{field}.input_contract_version"
|
|
812
|
+
),
|
|
813
|
+
}
|
|
814
|
+
if result["provider"] != "apify" or result["status"] != "SUCCEEDED":
|
|
815
|
+
raise ContractError(f"{field} must be a successful Apify run")
|
|
816
|
+
expected = CANONICAL_PROVIDER_RUNS.get(result["logical_key"])
|
|
817
|
+
if expected is None or result["actor_id"] != expected["actor_id"]:
|
|
818
|
+
raise ContractError(f"{field} is not a canonical organic trend run")
|
|
819
|
+
if result["input_contract_version"] != PROVIDER_INPUT_CONTRACT_VERSION:
|
|
820
|
+
raise ContractError(f"{field} input contract version mismatch")
|
|
821
|
+
input_keys = value.get("input_keys")
|
|
822
|
+
if not isinstance(input_keys, list) or input_keys != sorted(expected["input_keys"]):
|
|
823
|
+
raise ContractError(f"{field}.input_keys do not match the canonical adapter")
|
|
824
|
+
result["input_keys"] = input_keys
|
|
825
|
+
metadata = value.get("actor_metadata")
|
|
826
|
+
if not isinstance(metadata, dict):
|
|
827
|
+
raise ContractError(f"{field}.actor_metadata must be an object")
|
|
828
|
+
result["actor_metadata"] = {
|
|
829
|
+
"path": require_string(metadata.get("path"), f"{field}.actor_metadata.path"),
|
|
830
|
+
"sha256": require_sha256(
|
|
831
|
+
metadata.get("sha256"), f"{field}.actor_metadata.sha256"
|
|
832
|
+
),
|
|
833
|
+
}
|
|
834
|
+
return result
|
|
835
|
+
|
|
836
|
+
|
|
837
|
+
def validate_raw_manifest(root: Path) -> dict[str, Any]:
|
|
838
|
+
manifest_path = root / "raw/manifest.json"
|
|
839
|
+
value = read_json(manifest_path)
|
|
840
|
+
if value.get("schema_version") != SCHEMA_VERSION:
|
|
841
|
+
raise ContractError("raw manifest schema_version must be 1")
|
|
842
|
+
require_string(
|
|
843
|
+
value.get("collection_epoch"), "raw manifest collection_epoch"
|
|
844
|
+
)
|
|
845
|
+
started = parse_rfc3339(value.get("collection_started_at"), "collection_started_at")
|
|
846
|
+
completed = parse_rfc3339(
|
|
847
|
+
value.get("collection_completed_at"), "collection_completed_at"
|
|
848
|
+
)
|
|
849
|
+
if completed < started:
|
|
850
|
+
raise ContractError("collection_completed_at precedes collection_started_at")
|
|
851
|
+
|
|
852
|
+
accounts = validate_accounts(value.get("accounts"), "raw manifest accounts")
|
|
853
|
+
file_accounts, accounts_source = validate_accounts_file(root)
|
|
854
|
+
if value.get("accounts_source") != accounts_source:
|
|
855
|
+
raise ContractError("raw manifest accounts_source hash mismatch")
|
|
856
|
+
if accounts != file_accounts:
|
|
857
|
+
raise ContractError("raw manifest accounts differ from raw/accounts.json")
|
|
858
|
+
|
|
859
|
+
datasets = value.get("datasets")
|
|
860
|
+
if not isinstance(datasets, list) or not datasets:
|
|
861
|
+
raise ContractError("raw manifest datasets must be a non-empty list")
|
|
862
|
+
dataset_shapes_seen: set[tuple[str, str]] = set()
|
|
863
|
+
for index, dataset in enumerate(datasets):
|
|
864
|
+
if not isinstance(dataset, dict):
|
|
865
|
+
raise ContractError(f"datasets[{index}] must be an object")
|
|
866
|
+
platform = require_string(dataset.get("platform"), f"datasets[{index}].platform")
|
|
867
|
+
if platform not in {"tiktok", "instagram"}:
|
|
868
|
+
raise ContractError(f"datasets[{index}].platform is unsupported")
|
|
869
|
+
kind = require_string(dataset.get("kind"), f"datasets[{index}].kind")
|
|
870
|
+
if kind not in {"discovery", "enrichment"}:
|
|
871
|
+
raise ContractError(f"datasets[{index}].kind is unsupported")
|
|
872
|
+
shape = (platform, kind)
|
|
873
|
+
logical_key = f"{platform}-{kind}"
|
|
874
|
+
if shape in dataset_shapes_seen:
|
|
875
|
+
raise ContractError(f"duplicate raw manifest dataset: {logical_key}")
|
|
876
|
+
dataset_shapes_seen.add(shape)
|
|
877
|
+
if logical_key not in CANONICAL_PROVIDER_RUNS:
|
|
878
|
+
raise ContractError(f"datasets[{index}] is not a canonical provider run")
|
|
879
|
+
if dataset.get("path") != f"raw/{logical_key}.json":
|
|
880
|
+
raise ContractError(
|
|
881
|
+
f"datasets[{index}].path must be raw/{logical_key}.json"
|
|
882
|
+
)
|
|
883
|
+
path = resolve_relative(root, dataset.get("path"), f"datasets[{index}].path")
|
|
884
|
+
expected = require_sha256(dataset.get("sha256"), f"datasets[{index}].sha256")
|
|
885
|
+
if not path.is_file() or sha256_file(path) != expected:
|
|
886
|
+
raise ContractError(f"datasets[{index}] file hash mismatch")
|
|
887
|
+
|
|
888
|
+
runs = value.get("provider_runs")
|
|
889
|
+
if not isinstance(runs, list) or not 1 <= len(runs) <= len(CANONICAL_PROVIDER_RUNS):
|
|
890
|
+
raise ContractError("raw manifest provider_runs must contain 1–4 runs")
|
|
891
|
+
logical_keys: set[str] = set()
|
|
892
|
+
run_ids: set[str] = set()
|
|
893
|
+
dataset_ids: set[str] = set()
|
|
894
|
+
for index, run in enumerate(runs):
|
|
895
|
+
canonical = validate_provider_run(run, f"provider_runs[{index}]")
|
|
896
|
+
metadata_path = resolve_relative(
|
|
897
|
+
root,
|
|
898
|
+
canonical["actor_metadata"]["path"],
|
|
899
|
+
f"provider_runs[{index}].actor_metadata.path",
|
|
900
|
+
)
|
|
901
|
+
if (
|
|
902
|
+
not metadata_path.is_file()
|
|
903
|
+
or sha256_file(metadata_path) != canonical["actor_metadata"]["sha256"]
|
|
904
|
+
):
|
|
905
|
+
raise ContractError(
|
|
906
|
+
f"provider_runs[{index}].actor_metadata hash mismatch"
|
|
907
|
+
)
|
|
908
|
+
expected_metadata_path = (
|
|
909
|
+
f"raw/{canonical['logical_key']}-actor-metadata.json"
|
|
910
|
+
)
|
|
911
|
+
if canonical["actor_metadata"]["path"] != expected_metadata_path:
|
|
912
|
+
raise ContractError(
|
|
913
|
+
f"provider_runs[{index}].actor_metadata.path must be "
|
|
914
|
+
f"{expected_metadata_path}"
|
|
915
|
+
)
|
|
916
|
+
if canonical["logical_key"] in logical_keys:
|
|
917
|
+
raise ContractError("raw manifest provider_runs contains a duplicate logical_key")
|
|
918
|
+
if canonical["run_id"] in run_ids or canonical["dataset_id"] in dataset_ids:
|
|
919
|
+
raise ContractError("raw manifest provider_runs contains duplicate provider IDs")
|
|
920
|
+
logical_keys.add(canonical["logical_key"])
|
|
921
|
+
run_ids.add(canonical["run_id"])
|
|
922
|
+
dataset_ids.add(canonical["dataset_id"])
|
|
923
|
+
run_shapes = {
|
|
924
|
+
(
|
|
925
|
+
CANONICAL_PROVIDER_RUNS[key]["platform"],
|
|
926
|
+
CANONICAL_PROVIDER_RUNS[key]["kind"],
|
|
927
|
+
)
|
|
928
|
+
for key in logical_keys
|
|
929
|
+
}
|
|
930
|
+
dataset_shapes = {
|
|
931
|
+
(dataset["platform"], dataset["kind"])
|
|
932
|
+
for dataset in datasets
|
|
933
|
+
if isinstance(dataset, dict)
|
|
934
|
+
}
|
|
935
|
+
if dataset_shapes != run_shapes:
|
|
936
|
+
raise ContractError("raw manifest datasets do not match canonical provider runs")
|
|
937
|
+
|
|
938
|
+
required_discovery_keys = {
|
|
939
|
+
f"{account['platform']}-discovery" for account in accounts
|
|
940
|
+
}
|
|
941
|
+
if not required_discovery_keys.issubset(logical_keys):
|
|
942
|
+
state = load_run_state(root)
|
|
943
|
+
missing = []
|
|
944
|
+
for key in sorted(required_discovery_keys - logical_keys):
|
|
945
|
+
run = state["provider_runs"].get(key)
|
|
946
|
+
if (not isinstance(run, dict)
|
|
947
|
+
or run.get("status") not in TERMINAL_PROVIDER_FAILURE_STATUSES
|
|
948
|
+
or not isinstance(run.get("run_id"), str)
|
|
949
|
+
or not run["run_id"].strip()):
|
|
950
|
+
missing.append(key)
|
|
951
|
+
if missing:
|
|
952
|
+
raise ContractError(
|
|
953
|
+
f"raw manifest is missing required discovery runs: {', '.join(missing)}"
|
|
954
|
+
)
|
|
955
|
+
|
|
956
|
+
linked_datasets = value.get("linked_datasets")
|
|
957
|
+
if not isinstance(linked_datasets, list):
|
|
958
|
+
raise ContractError("raw manifest linked_datasets must be a list")
|
|
959
|
+
expected_linked_count = 1 if "tiktok-enrichment" in logical_keys else 0
|
|
960
|
+
if len(linked_datasets) != expected_linked_count:
|
|
961
|
+
raise ContractError(
|
|
962
|
+
"raw manifest must contain exactly one TikTok comments bundle when "
|
|
963
|
+
"TikTok enrichment is present"
|
|
964
|
+
)
|
|
965
|
+
if linked_datasets:
|
|
966
|
+
linked = linked_datasets[0]
|
|
967
|
+
if not isinstance(linked, dict):
|
|
968
|
+
raise ContractError("raw manifest linked dataset must be an object")
|
|
969
|
+
if (
|
|
970
|
+
linked.get("kind") != TIKTOK_COMMENTS_LINK_KIND
|
|
971
|
+
or linked.get("parent_logical_key") != "tiktok-enrichment"
|
|
972
|
+
or linked.get("path") != TIKTOK_COMMENTS_BUNDLE_PATH
|
|
973
|
+
):
|
|
974
|
+
raise ContractError("raw manifest TikTok comments bundle identity is invalid")
|
|
975
|
+
linked_path = resolve_relative(
|
|
976
|
+
root,
|
|
977
|
+
linked.get("path"),
|
|
978
|
+
"linked_datasets[0].path",
|
|
979
|
+
)
|
|
980
|
+
linked_hash = require_sha256(
|
|
981
|
+
linked.get("sha256"),
|
|
982
|
+
"linked_datasets[0].sha256",
|
|
983
|
+
)
|
|
984
|
+
if not linked_path.is_file() or sha256_file(linked_path) != linked_hash:
|
|
985
|
+
raise ContractError("raw manifest TikTok comments bundle hash mismatch")
|
|
986
|
+
main_dataset = read_json(root / "raw/tiktok-enrichment.json")
|
|
987
|
+
bundle = read_json(linked_path)
|
|
988
|
+
validate_linked_tiktok_comments_bundle(main_dataset, bundle)
|
|
989
|
+
|
|
990
|
+
return value
|
|
991
|
+
|
|
992
|
+
|
|
993
|
+
def validate_collection_fresh(manifest: dict[str, Any]) -> None:
|
|
994
|
+
"""Validate the collection clock, not a methodological expiry date."""
|
|
995
|
+
completed = parse_rfc3339(
|
|
996
|
+
manifest.get("collection_completed_at"), "collection_completed_at"
|
|
997
|
+
)
|
|
998
|
+
age_seconds = (datetime.now(timezone.utc) - completed).total_seconds()
|
|
999
|
+
if age_seconds < -300:
|
|
1000
|
+
raise ContractError("collection completion time is in the future")
|
|
1001
|
+
|
|
1002
|
+
|
|
1003
|
+
def validate_manifest_against_state(
|
|
1004
|
+
root: Path, manifest: dict[str, Any], state: dict[str, Any]
|
|
1005
|
+
) -> None:
|
|
1006
|
+
from _provider_inputs import build_discovery_inputs, validate_collection_window
|
|
1007
|
+
|
|
1008
|
+
context = validate_context(root / "context.json")
|
|
1009
|
+
window = validate_collection_window(context, state.get("collection_window"))
|
|
1010
|
+
if manifest.get("collection_started_at") != window["reference_at"]:
|
|
1011
|
+
raise ContractError("raw manifest collection start differs from the first attempt")
|
|
1012
|
+
discovery_inputs = build_discovery_inputs(manifest["accounts"], context, window)
|
|
1013
|
+
if manifest.get("collection_epoch") != state.get("collection_epoch"):
|
|
1014
|
+
raise ContractError("raw manifest belongs to a different collection epoch")
|
|
1015
|
+
if manifest.get("accounts_source") != state.get("accounts_source"):
|
|
1016
|
+
raise ContractError("raw manifest accounts source differs from run-state")
|
|
1017
|
+
manifest_runs = {
|
|
1018
|
+
item["logical_key"]: item
|
|
1019
|
+
for item in (
|
|
1020
|
+
validate_provider_run(run, f"manifest.provider_runs[{index}]")
|
|
1021
|
+
for index, run in enumerate(manifest["provider_runs"])
|
|
1022
|
+
)
|
|
1023
|
+
}
|
|
1024
|
+
completed_state_runs = {
|
|
1025
|
+
key: value
|
|
1026
|
+
for key, value in state["provider_runs"].items()
|
|
1027
|
+
if isinstance(value, dict) and value.get("status") == "SUCCEEDED"
|
|
1028
|
+
}
|
|
1029
|
+
if set(manifest_runs) != set(completed_state_runs):
|
|
1030
|
+
raise ContractError("raw manifest provider runs differ from completed run-state handles")
|
|
1031
|
+
datasets = {
|
|
1032
|
+
(item["platform"], item["kind"]): item for item in manifest["datasets"]
|
|
1033
|
+
}
|
|
1034
|
+
for key, manifest_run in manifest_runs.items():
|
|
1035
|
+
state_run = completed_state_runs[key]
|
|
1036
|
+
for field in (
|
|
1037
|
+
"actor_id",
|
|
1038
|
+
"run_id",
|
|
1039
|
+
"dataset_id",
|
|
1040
|
+
"status",
|
|
1041
|
+
"input_sha256",
|
|
1042
|
+
"input_contract_version",
|
|
1043
|
+
"input_keys",
|
|
1044
|
+
"actor_metadata",
|
|
1045
|
+
):
|
|
1046
|
+
if state_run.get(field) != manifest_run.get(field):
|
|
1047
|
+
raise ContractError(f"provider run {key} differs from run-state field {field}")
|
|
1048
|
+
input_path = resolve_relative(root, state_run.get("input_path"), f"{key}.input_path")
|
|
1049
|
+
if state_run.get("input_path") != f"raw/{key}-input.json":
|
|
1050
|
+
raise ContractError(f"provider run {key} input path is not canonical")
|
|
1051
|
+
if not input_path.is_file() or sha256_file(input_path) != manifest_run["input_sha256"]:
|
|
1052
|
+
raise ContractError(f"provider run {key} input hash mismatch")
|
|
1053
|
+
if key in discovery_inputs and read_json(input_path) != discovery_inputs[key]:
|
|
1054
|
+
raise ContractError(f"provider run {key} input differs from the frozen collection scope")
|
|
1055
|
+
expected_idempotency_key = provider_idempotency_key(
|
|
1056
|
+
state["context_sha256"],
|
|
1057
|
+
state["collection_epoch"],
|
|
1058
|
+
key,
|
|
1059
|
+
manifest_run["actor_id"],
|
|
1060
|
+
manifest_run["input_sha256"],
|
|
1061
|
+
)
|
|
1062
|
+
if state_run.get("idempotency_key") != expected_idempotency_key:
|
|
1063
|
+
raise ContractError(f"provider run {key} idempotency key mismatch")
|
|
1064
|
+
spec = CANONICAL_PROVIDER_RUNS[key]
|
|
1065
|
+
dataset = datasets[(spec["platform"], spec["kind"])]
|
|
1066
|
+
if state_run.get("raw_path") != dataset["path"] or state_run.get("raw_sha256") != dataset["sha256"]:
|
|
1067
|
+
raise ContractError(f"provider run {key} raw dataset differs from run-state")
|
|
1068
|
+
|
|
1069
|
+
|
|
1070
|
+
def provider_idempotency_key(
|
|
1071
|
+
context_hash: str,
|
|
1072
|
+
collection_epoch: str,
|
|
1073
|
+
key: str,
|
|
1074
|
+
actor_id: str,
|
|
1075
|
+
input_hash: str,
|
|
1076
|
+
) -> str:
|
|
1077
|
+
raw = (
|
|
1078
|
+
"organic-trend-watch:"
|
|
1079
|
+
f"{context_hash}:{collection_epoch}:{key}:{actor_id}:{input_hash}"
|
|
1080
|
+
)
|
|
1081
|
+
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
|
1082
|
+
|
|
1083
|
+
|
|
1084
|
+
def new_run_state(root: Path) -> dict[str, Any]:
|
|
1085
|
+
context_path = root / "context.json"
|
|
1086
|
+
validate_context(context_path)
|
|
1087
|
+
now = utc_now()
|
|
1088
|
+
return {
|
|
1089
|
+
"schema_version": SCHEMA_VERSION,
|
|
1090
|
+
"section_id": SECTION_ID,
|
|
1091
|
+
"analysis_contract_version": ANALYSIS_CONTRACT_VERSION,
|
|
1092
|
+
"context_sha256": sha256_file(context_path),
|
|
1093
|
+
"collection_epoch": uuid.uuid4().hex,
|
|
1094
|
+
"status": "running",
|
|
1095
|
+
"created_at": now,
|
|
1096
|
+
"updated_at": now,
|
|
1097
|
+
"steps": {name: {"status": "pending"} for name in STEPS},
|
|
1098
|
+
"provider_runs": {},
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
|
|
1102
|
+
def load_run_state(root: Path) -> dict[str, Any]:
|
|
1103
|
+
value = read_json(root / "run-state.json")
|
|
1104
|
+
if value.get("schema_version") != SCHEMA_VERSION:
|
|
1105
|
+
raise ContractError("run-state schema_version must be 1")
|
|
1106
|
+
if value.get("section_id") != SECTION_ID:
|
|
1107
|
+
raise ContractError(f"run-state section_id must be {SECTION_ID}")
|
|
1108
|
+
if value.get("analysis_contract_version") != ANALYSIS_CONTRACT_VERSION:
|
|
1109
|
+
raise ContractError("run-state analysis contract version mismatch")
|
|
1110
|
+
expected = sha256_file(root / "context.json")
|
|
1111
|
+
if value.get("context_sha256") != expected:
|
|
1112
|
+
raise ContractError("run-state belongs to a different context.json")
|
|
1113
|
+
require_string(value.get("collection_epoch"), "run-state.collection_epoch")
|
|
1114
|
+
if not isinstance(value.get("steps"), dict) or not isinstance(
|
|
1115
|
+
value.get("provider_runs"), dict
|
|
1116
|
+
):
|
|
1117
|
+
raise ContractError("run-state has invalid steps or provider_runs")
|
|
1118
|
+
if set(value["steps"]) != set(STEPS):
|
|
1119
|
+
raise ContractError("run-state must contain exactly the canonical steps")
|
|
1120
|
+
for step, step_state in value["steps"].items():
|
|
1121
|
+
if not isinstance(step_state, dict) or step_state.get("status") not in {
|
|
1122
|
+
"pending",
|
|
1123
|
+
"running",
|
|
1124
|
+
"completed",
|
|
1125
|
+
"failed",
|
|
1126
|
+
}:
|
|
1127
|
+
raise ContractError(f"run-state step {step} has an invalid status")
|
|
1128
|
+
return value
|
|
1129
|
+
|
|
1130
|
+
|
|
1131
|
+
def save_run_state(root: Path, state: dict[str, Any]) -> None:
|
|
1132
|
+
state["updated_at"] = utc_now()
|
|
1133
|
+
atomic_write_json(root / "run-state.json", state)
|