edsger 0.60.0 → 0.62.0

package/dist/phases/sync-org-repos/index.js

@@ -0,0 +1,143 @@
+/**
+ * Phase: sync-org-repos
+ *
+ * Fetches all repositories from a GitHub organization using the local `gh` CLI,
+ * then creates a product for each repo that isn't already linked to one
+ * within the same team.
+ *
+ * Uses `gh api --paginate` for truly unlimited pagination (no hardcoded cap).
+ * Forks and archived repos are filtered out client-side.
+ */
+import { execFile } from 'child_process';
+import { promisify } from 'util';
+import { getSupabase, hasSupabaseSession } from '../../supabase/client.js';
+import { logDebug, logError, logInfo, logWarning } from '../../utils/logger.js';
+const execFileAsync = promisify(execFile);
+const ORG_NAME_RE = /^[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?$/;
+async function fetchOrgRepos(orgLogin, verbose) {
+    logInfo(`Fetching repos for org "${orgLogin}" via gh CLI...`);
+    const { stdout } = await execFileAsync('gh', [
+        'api',
+        `--paginate`,
+        `/orgs/${encodeURIComponent(orgLogin)}/repos?per_page=100&type=sources&sort=updated`,
+        '--jq',
+        '.[] | select(.archived == false) | {name, full_name, description, html_url}',
+    ], { timeout: 120_000 });
+    if (!stdout.trim()) {
+        return [];
+    }
+    const repos = [];
+    for (const line of stdout.trim().split('\n')) {
+        if (!line)
+            continue;
+        try {
+            repos.push(JSON.parse(line));
+        }
+        catch {
+            // skip malformed lines
+        }
+    }
+    if (verbose) {
+        logDebug(`Fetched ${repos.length} source repos from ${orgLogin}`);
+    }
+    return repos;
+}
+export async function syncOrgRepos(opts) {
+    const { teamId, orgLogin, userId, verbose } = opts;
+    if (!ORG_NAME_RE.test(orgLogin) || orgLogin.length > 39) {
+        return {
+            status: 'error',
+            message: `Invalid GitHub organization name: "${orgLogin}"`,
+            total: 0,
+            created: 0,
+            skipped: 0,
+        };
+    }
+    if (!hasSupabaseSession()) {
+        return {
+            status: 'error',
+            message: 'Supabase session unavailable. Sign in to the Edsger desktop app.',
+            total: 0,
+            created: 0,
+            skipped: 0,
+        };
+    }
+    const supabase = getSupabase();
+    let orgRepos;
+    try {
+        orgRepos = await fetchOrgRepos(orgLogin, verbose);
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        logError(`Failed to fetch repos: ${msg}`);
+        return {
+            status: 'error',
+            message: `Failed to fetch repos from GitHub CLI: ${msg}`,
+            total: 0,
+            created: 0,
+            skipped: 0,
+        };
+    }
+    if (orgRepos.length === 0) {
+        logWarning(`No repos found in org "${orgLogin}"`);
+        return {
+            status: 'success',
+            message: `No repositories found in ${orgLogin}`,
+            total: 0,
+            created: 0,
+            skipped: 0,
+        };
+    }
+    logInfo(`Found ${orgRepos.length} repos in ${orgLogin}`);
+    const repoFullNames = orgRepos.map((r) => r.full_name);
+    const { data: existing } = await supabase
+        .from('products')
+        .select('github_repository_full_name')
+        .eq('team_id', teamId)
+        .in('github_repository_full_name', repoFullNames);
+    const linkedRepos = new Set((existing || []).map((p) => p.github_repository_full_name));
+    const toCreate = orgRepos.filter((r) => !linkedRepos.has(r.full_name));
+    if (toCreate.length === 0) {
+        const msg = `All ${linkedRepos.size} repos already linked to products`;
+        logInfo(msg);
+        return {
+            status: 'success',
+            message: msg,
+            total: orgRepos.length,
+            created: 0,
+            skipped: linkedRepos.size,
+        };
+    }
+    if (verbose) {
+        for (const r of toCreate) {
+            logDebug(`  Will create product: ${r.full_name}`);
+        }
+    }
+    const rows = toCreate.map((repo) => ({
+        name: repo.name,
+        description: repo.description,
+        team_id: teamId,
+        created_by: userId,
+        github_repository_full_name: repo.full_name,
+        github_repository_url: repo.html_url,
+    }));
+    const { error: insertError } = await supabase.from('products').insert(rows);
+    if (insertError) {
+        logError(`Failed to insert products: ${insertError.message}`);
+        return {
+            status: 'error',
+            message: `Failed to create products: ${insertError.message}`,
+            total: orgRepos.length,
+            created: 0,
+            skipped: linkedRepos.size,
+        };
+    }
+    return {
+        status: 'success',
+        message: `Created ${toCreate.length} products, ${linkedRepos.size} already existed`,
+        total: orgRepos.length,
+        created: toCreate.length,
+        skipped: linkedRepos.size,
+        repos: toCreate.map((r) => r.full_name),
+    };
+}

package/dist/skills/phase/data-flow/SKILL.md

@@ -0,0 +1,82 @@
+---
+description: Map a product's data nodes (sources, datasets, transforms, sinks, queues, models) and the connections between them into a structured data flow
+kind: phase
+user-invocable: false
+---
+
+You are a senior data engineer mapping a codebase's **data flow** — where data originates, what stores it, what computes on it, where it terminates, and how it moves between those nodes. Your output is a structured graph that the desktop app will render as a flow diagram. You are NOT inspecting a running system or its production data — you are reading source code and producing a structured description of each node and edge.
+
+**What counts as a data node**:
+
+- `source` — external input: API ingestion job, scraper, webhook handler that produces a record, sensor reading, manual upload form
+- `dataset` — stored collection: DB table (Postgres / SQLite / Mongo / DynamoDB row class), object-store bucket (S3 / GCS), file on disk, in-memory cache (Redis hash, in-process store), Parquet / CSV / JSONL on disk
+- `transform` — computation: ETL stage, data normalizer, scorer, aggregation job, pipeline step, scheduled job, queue worker that mutates payloads
+- `sink` — terminal output: external API write (Stripe, Slack, email), generated report, alert, file export
+- `queue` — async message bus: SQS / Kafka topic / NATS subject / Redis pub/sub channel / in-memory event emitter
+- `model` — ML model or LLM/scoring service: OpenAI / Anthropic / Gemini call site, local model invocation, ranking model. Treat as a special transform when its primary role is producing a prediction or embedding.
+
+**Distinguish from screen-flow**: this is about **data movement**, not user navigation. UI components are not data nodes. A button-click handler that *also* calls an API is itself a source / transform / sink trigger, not a "screen".
+
+**For each node, extract a DataNodeSchema** with these fields:
+
+- `slug` — stable short identifier (kebab-case, e.g. `raw-events`, `enrich-user`)
+- `name` — human-readable display name
+- `kind` — one of the six above
+- `file` — primary source file path relative to repo root (schema/migration file for datasets, definition file for transforms/sinks/sources/queues, model invocation file for models)
+- `description?` — one short sentence ("Nightly job that scrapes vendor catalogs and normalizes them into the products table")
+- `tech?` — technology / format hint: `postgres` / `sqlite` / `parquet` / `s3` / `kafka` / `sqs` / `redis-pubsub` / `openai-api` / `anthropic-claude` / `gemini` / `bullmq` / `cron` / `playwright` / etc. Free-form, prefer the most common name
+- `schedule?` — for transforms / sources: `cron 0 0 * * *`, `on-event`, `manual`, `continuous`, `on-webhook`
+- `inputs?` — array of `{ name, type?, required?, description? }`. For transforms: what fields it reads. For sinks: what fields it sends. For models: prompt template variables / feature names
+- `outputs?` — array of `{ name, type?, required?, description? }`. For sources: what fields it emits. For datasets: column schema. For transforms: emitted record fields. For queues: message payload fields. For models: prediction / embedding fields
+- `sample?` — `{ columns: [string], rows: [[string]] }` — at most 4 sample rows for datasets only. Fabricate realistic placeholder content; this is a documentation artifact
+- `stats?` — array of `{ label, value }` for volume / latency / count hints (e.g. `{ label: "rows", value: "~2M" }`, `{ label: "p50 latency", value: "180ms" }`). Best-effort, leave empty if nothing useful is visible from code
+
+**Connections (edges)**: direction is always **data movement**. `fromSlug` is upstream (origin), `toSlug` is downstream (destination). Sources to extract from:
+
+- Database access calls: `db.query(...)`, ORM model calls (Prisma / SQLAlchemy / TypeORM / ActiveRecord), Supabase / Firestore SDK calls, raw SQL strings referencing a known table
+- Object-store reads/writes: `s3.getObject` / `s3.putObject`, `fs.readFile` / `fs.writeFile` on a known dataset path
+- Queue / topic publishes & consumes: `producer.send(...)`, `subscribe('topic-name', ...)`, BullMQ `queue.add` / `worker.process`
+- Model invocations: SDK calls like `anthropic.messages.create`, `openai.chat.completions.create`, local model `predict(...)` — wire the calling transform / sink to the model node
+- Cron / scheduler triggers: cron config → `control` edge from a synthetic `cron` node to the job, OR represent cron schedule on the job's `schedule` field if there's no other reason to model the trigger separately
+- External API calls (HTTP fetch to a third-party): emit a sink node for the third party and a `data` edge to it
+
+For each edge produce:
+
+- `fromSlug` — upstream node's slug
+- `toSlug` — downstream node's slug
+- `kind`:
+  - `data` — plain data movement (most common: transform reads dataset, transform writes to sink)
+  - `event` — async event/message passed via a queue or pub/sub
+  - `control` — control-flow trigger without data payload (cron triggers job, file-watcher triggers handler)
+  - `derives` — lineage: downstream node is materialized from upstream (rollup, materialized view, snapshot)
+- `label?` — free-form descriptor (`nightly batch`, `on user signup`, `embedding`, `daily rollup`)
+- `sourceFile?` — file containing the connection definition (when distinct from the from-node's file)
+
+**Discipline**:
+
+- Be grounded — every node MUST correspond to actual code (a table, a file, a queue, a model invocation, etc). No invented datasets.
+- Deduplicate: if multiple files reference the same dataset / queue / model, keep one node and emit edges from each consumer.
+- Prefer fewer, clearer nodes. If the system has > 40 data nodes, pick the most important 30 and note skipped count in the summary.
+- Datasets, queues, and models are nouns; transforms and sources are verbs. Name them accordingly.
+- Edges always point to a node you also emit. Drop any edge whose target you couldn't extract.
+- A model invocation is its own node, not part of the calling transform — this lets a reader see "all the places we call Claude" at a glance.
+
+**Process**:
+
+<!-- if:hasCodebase -->
+
+1. **Detect the stack**: Read `package.json` / `pyproject.toml` / `go.mod` / `Cargo.toml` / `requirements.txt` / `Gemfile` to identify the runtime and obvious data libraries.
+2. **Enumerate datasets**: scan migrations / schema files / ORM models / dbt models / table-defining SQL files. Each table or collection becomes a `dataset` node.
+3. **Enumerate queues / topics / models**: search for queue config (BullMQ, Kafka, SQS, NATS) and model SDK imports (`anthropic`, `openai`, `@google/genai`). Each becomes a `queue` or `model` node.
+4. **Enumerate sources & sinks**: ingestion scripts (scrapers, webhook handlers, file watchers, cron-driven importers) → `source` nodes; outbound integrations (email senders, Stripe writes, Slack senders, third-party API POSTs) → `sink` nodes.
+5. **Enumerate transforms**: pipeline files, ETL scripts, queue worker functions, scheduled jobs, normalizers, aggregators. Each becomes a `transform` node.
+6. **Wire edges**: for each transform / source / sink / queue handler / model call, read just enough of its body to identify what it reads from and writes to, then emit edges. Use `data` for plain reads/writes, `event` when the carrier is a queue, `control` for triggers without data, `derives` for materialized rollups.
+7. **Compose the summary**: 1-3 sentences describing what kind of system this is and its primary pipelines.
+
+<!-- endif -->
+<!-- if:!hasCodebase -->
+
+8. **Use the provided context** (product description and any user guidance) to infer reasonable data nodes for the system's domain. Be explicit in the summary that the flow is inferred rather than extracted.
+9. Each inferred node should still be a complete DataNodeSchema with concrete labels and sample content — no placeholder brackets.
+
+<!-- endif -->

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "edsger",
-  "version": "0.60.0",
+  "version": "0.62.0",
   "type": "module",
   "bin": {
     "edsger": "dist/index.js"
@@ -50,8 +50,8 @@
     "commander": "^12.0.0",
     "cosmiconfig": "^9.0.0",
     "dotenv": "^16.4.5",
-    "edsger-contract": "0.4.0",
-    "edsger-tools": "0.4.0",
+    "edsger-contract": "0.5.0",
+    "edsger-tools": "0.5.0",
     "gray-matter": "^4.0.3",
     "zod": "^4.0.0"
   },