@topy-ai/maggie 0.7.32 → 0.7.34

package/bundled-templates/astro-blog/starter/src/pages/api/ops/migrations/import.ts

@@ -0,0 +1,458 @@
+import type { APIRoute } from "astro";
+import { createHash } from "node:crypto";
+import { audit, db, id, now } from "../../../../lib/db";
+import { json } from "../../../../lib/ops-auth";
+import {
+  checksum,
+  parseMigration,
+  type MigrationFormat,
+  type MigrationRecord,
+} from "../../../../lib/migration";
+
+function format(value: unknown): MigrationFormat | undefined {
+  return ["json", "csv", "wxr", "sitemap", "media-archive"].includes(
+    String(value),
+  )
+    ? (String(value) as MigrationFormat)
+    : undefined;
+}
+function source(value: unknown): string | undefined {
+  try {
+    const url = new URL(String(value));
+    return /^https?:$/.test(url.protocol) ? url.toString() : undefined;
+  } catch {
+    return undefined;
+  }
+}
+function sourceChecksum(records: MigrationRecord[]): string {
+  return createHash("sha256")
+    .update(
+      records
+        .map((record) => checksum(record))
+        .sort()
+        .join("\n"),
+    )
+    .digest("hex");
+}
+function metadata(record: MigrationRecord): string {
+  const raw = record.seo?.custom_metadata_json || "{}";
+  try {
+    const parsed = JSON.parse(raw);
+    return JSON.stringify(parsed && typeof parsed === "object" ? parsed : {});
+  } catch {
+    return "{}";
+  }
+}
+
+export const POST: APIRoute = async ({ request }) => {
+  const body = (await request.json().catch(() => null)) as any;
+  const kind = format(body?.format);
+  if (!kind)
+    return json(
+      { error: "format must be json, csv, wxr, sitemap, or media-archive" },
+      400,
+    );
+  let input = typeof body?.input === "string" ? body.input : "";
+  const location = source(body?.sourceUrl);
+  if (!input && location) {
+    const response = await fetch(location, {
+      signal: AbortSignal.timeout(30000),
+    });
+    if (!response.ok)
+      return json({ error: `source returned ${response.status}` }, 502);
+    input = await response.text();
+  }
+  if (!input) return json({ error: "input or sourceUrl is required" }, 400);
+  try {
+    const records = parseMigration(input, kind, location);
+    const migrationId = id("migration");
+    const timestamp = now();
+    const mode = body?.mode === "apply" ? "apply" : "preview";
+    const aggregateChecksum = sourceChecksum(records);
+    const syncMode = ["all", "new", "modified"].includes(body?.syncMode)
+      ? body.syncMode
+      : body?.incremental === true
+        ? "modified"
+        : "all";
+    const conflict = ["source-wins", "target-wins", "manual"].includes(
+      body?.conflict,
+    )
+      ? body.conflict
+      : "manual";
+    const options = {
+      format: kind,
+      incremental: syncMode !== "all",
+      syncMode,
+      detectDeleted: body?.detectDeleted === true,
+      conflict,
+    };
+    db()
+      .prepare(
+        "INSERT INTO wp_migrations (id,source_url,mode,status,options_json,counts_json,started_at,source_checksum) VALUES (?,?,?,?,?,?,?,?)",
+      )
+      .run(
+        migrationId,
+        location || "inline",
+        mode,
+        "running",
+        JSON.stringify(options),
+        "{}",
+        timestamp,
+        aggregateChecksum,
+      );
+    db()
+      .prepare(
+        "INSERT INTO migration_sources (id,kind,location,options_json,checksum,created_at,updated_at) VALUES (?,?,?,?,?,?,?)",
+      )
+      .run(
+        id("migration_source"),
+        kind,
+        location || "inline",
+        JSON.stringify(body || {}),
+        aggregateChecksum,
+        timestamp,
+        timestamp,
+      );
+    if (mode === "preview") {
+      const preview = records
+        .slice(0, 20)
+        .map((record) => ({ ...record, checksum: checksum(record) }));
+      db()
+        .prepare(
+          "UPDATE wp_migrations SET status='completed',counts_json=?,completed_at=?,checkpoint_json=? WHERE id=?",
+        )
+        .run(
+          JSON.stringify({
+            total: records.length,
+            previewed: preview.length,
+            sourceChecksum: aggregateChecksum,
+            syncMode,
+            detectDeleted: options.detectDeleted,
+          }),
+          now(),
+          JSON.stringify({
+            processed: 0,
+            total: records.length,
+            completed: false,
+          }),
+          migrationId,
+        );
+      return json({
+        id: migrationId,
+        mode,
+        total: records.length,
+        sourceChecksum: aggregateChecksum,
+        syncMode,
+        items: preview,
+      });
+    }
+    const authorId = "author-imported";
+    db()
+      .prepare(
+        "INSERT OR IGNORE INTO authors (id,slug,name,credentials_json,created_at,updated_at) VALUES (?,?,?,?,?,?)",
+      )
+      .run(authorId, "imported", "Imported author", "[]", timestamp, timestamp);
+    const write = db().transaction(() => {
+      let created = 0;
+      let updated = 0;
+      let skipped = 0;
+      let conflicts = 0;
+      let deleted = 0;
+      const seen = new Set<string>();
+      const writeItem = db().prepare(
+        "INSERT INTO wp_import_items (id,migration_id,entity_type,source_id,target_id,action,error,created_at) VALUES (?,?,?,?,?,?,?,?) ON CONFLICT(migration_id,entity_type,source_id) DO UPDATE SET target_id=excluded.target_id,action=excluded.action,error=excluded.error",
+      );
+      const conflictInsert = db().prepare(
+        "INSERT INTO migration_conflicts (id,migration_id,source_id,target_id,resolution,details_json,created_at) VALUES (?,?,?,?,?,?,?)",
+      );
+      for (const record of records) {
+        const sourceId = `${kind}:${record.sourceId}`;
+        seen.add(sourceId);
+        if (kind === "media-archive") {
+          const media = record.seo?.media_archive
+            ? JSON.parse(record.seo.media_archive)
+            : {};
+          const mediaId = id("media");
+          db()
+            .prepare(
+              "INSERT INTO media_assets (id,source,source_id,source_url,local_url,mime_type,alt_text,caption,width,height,metadata_json,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source,source_id) DO UPDATE SET local_url=excluded.local_url,mime_type=excluded.mime_type,alt_text=excluded.alt_text,caption=excluded.caption,width=excluded.width,height=excluded.height,metadata_json=excluded.metadata_json,updated_at=excluded.updated_at",
+            )
+            .run(
+              mediaId,
+              "media-archive",
+              String(record.sourceId),
+              media.sourceUrl || record.canonicalUrl || "",
+              media.localUrl || null,
+              media.mimeType || null,
+              media.altText || record.title,
+              media.caption || "",
+              media.width || null,
+              media.height || null,
+              JSON.stringify(media),
+              timestamp,
+              timestamp,
+            );
+          writeItem.run(
+            id("import_item"),
+            migrationId,
+            "media",
+            sourceId,
+            mediaId,
+            "updated",
+            null,
+            timestamp,
+          );
+          updated++;
+          continue;
+        }
+        const existing = db()
+          .prepare("SELECT * FROM posts WHERE source='cms' AND source_id=?")
+          .get(sourceId) as any;
+        const recordChecksum = checksum(record);
+        if (
+          existing?.checksum === recordChecksum ||
+          (syncMode === "new" && existing) ||
+          (syncMode === "modified" &&
+            existing &&
+            record.sourceModified &&
+            existing.source_modified_at === record.sourceModified)
+        ) {
+          skipped++;
+          writeItem.run(
+            id("import_item"),
+            migrationId,
+            "post",
+            sourceId,
+            existing?.id || null,
+            "skipped",
+            "unchanged or excluded by sync mode",
+            timestamp,
+          );
+          continue;
+        }
+        if (
+          existing &&
+          conflict !== "source-wins" &&
+          existing.updated_at > (record.sourceModified || timestamp)
+        ) {
+          conflicts++;
+          conflictInsert.run(
+            id("conflict"),
+            migrationId,
+            sourceId,
+            existing.id,
+            conflict,
+            JSON.stringify({
+              sourceModified: record.sourceModified || null,
+              targetUpdated: existing.updated_at,
+            }),
+            timestamp,
+          );
+          if (conflict === "manual" || conflict === "target-wins") {
+            skipped++;
+            writeItem.run(
+              id("import_item"),
+              migrationId,
+              "post",
+              sourceId,
+              existing.id,
+              "skipped",
+              `${conflict} conflict requires review`,
+              timestamp,
+            );
+            continue;
+          }
+        }
+        const owner = db()
+          .prepare("SELECT id FROM posts WHERE slug=?")
+          .get(record.slug) as { id: string } | undefined;
+        const targetSlug =
+          owner && owner.id !== existing?.id
+            ? `${record.slug}-${record.sourceId}`
+            : record.slug;
+        const canonical =
+          record.canonicalUrl ||
+          new URL(
+            `/blog/${targetSlug}`,
+            import.meta.env.PUBLIC_SITE_URL || "https://example.com",
+          ).toString();
+        const seo = record.seo || {};
+        const values = {
+          slug: targetSlug,
+          title: record.title,
+          excerpt: record.excerpt,
+          html: record.html,
+          canonical,
+          status: record.publishedAt ? "published" : "draft",
+          publishedAt: record.publishedAt || null,
+          sourceModified: record.sourceModified || null,
+          seoTitle: seo.seo_title || null,
+          metaDescription: seo.meta_description || null,
+          focusKeyword: seo.focus_keyword || null,
+          canonicalOverride: seo.canonical_override || null,
+          robotsIndex: seo.robots_index || "index",
+          robotsFollow: seo.robots_follow || "follow",
+          ogImageUrl: seo.og_image_url || null,
+          schemaType: seo.schema_type || "Article",
+          customMetadata: metadata(record),
+          checksum: recordChecksum,
+        };
+        let targetId = existing?.id || id("post");
+        if (existing) {
+          const version =
+            Number(
+              (
+                db()
+                  .prepare(
+                    "SELECT MAX(version) version FROM post_revisions WHERE post_id=?",
+                  )
+                  .get(targetId) as any
+              )?.version || 0,
+            ) + 1;
+          db()
+            .prepare(
+              "INSERT INTO post_revisions (id,post_id,version,snapshot_json,actor_id,reason,created_at) VALUES (?,?,?,?,?,?,?)",
+            )
+            .run(
+              id("revision"),
+              targetId,
+              version,
+              JSON.stringify(existing),
+              "generic-migration",
+              "Generic migration",
+            );
+          db()
+            .prepare(
+              "UPDATE posts SET slug=?,title=?,excerpt=?,content_html=?,canonical_url=?,status=?,published_at=?,updated_at=?,source_modified_at=?,seo_title=?,meta_description=?,focus_keyword=?,canonical_override=?,robots_index=?,robots_follow=?,og_image_url=?,schema_type=?,custom_metadata_json=?,checksum=? WHERE id=?",
+            )
+            .run(
+              values.slug,
+              values.title,
+              values.excerpt,
+              values.html,
+              values.canonical,
+              values.status,
+              values.publishedAt,
+              timestamp,
+              values.sourceModified,
+              values.seoTitle,
+              values.metaDescription,
+              values.focusKeyword,
+              values.canonicalOverride,
+              values.robotsIndex,
+              values.robotsFollow,
+              values.ogImageUrl,
+              values.schemaType,
+              values.customMetadata,
+              values.checksum,
+              targetId,
+            );
+          updated++;
+        } else {
+          db()
+            .prepare(
+              "INSERT INTO posts (id,source,source_id,slug,title,excerpt,content_html,canonical_url,author_id,status,published_at,updated_at,created_at,source_modified_at,seo_title,meta_description,focus_keyword,canonical_override,robots_index,robots_follow,og_image_url,schema_type,custom_metadata_json,checksum) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+            )
+            .run(
+              targetId,
+              "cms",
+              sourceId,
+              values.slug,
+              values.title,
+              values.excerpt,
+              values.html,
+              values.canonical,
+              authorId,
+              values.status,
+              values.publishedAt,
+              timestamp,
+              timestamp,
+              values.sourceModified,
+              values.seoTitle,
+              values.metaDescription,
+              values.focusKeyword,
+              values.canonicalOverride,
+              values.robotsIndex,
+              values.robotsFollow,
+              values.ogImageUrl,
+              values.schemaType,
+              values.customMetadata,
+              values.checksum,
+            );
+          created++;
+        }
+        writeItem.run(
+          id("import_item"),
+          migrationId,
+          "post",
+          sourceId,
+          targetId,
+          existing ? "updated" : "created",
+          null,
+          timestamp,
+        );
+      }
+      if (options.detectDeleted && kind !== "media-archive") {
+        const candidates = db()
+          .prepare(
+            "SELECT id,source_id FROM posts WHERE source='cms' AND source_id LIKE ?",
+          )
+          .all(`${kind}:%`) as Array<{ id: string; source_id: string }>;
+        for (const candidate of candidates)
+          if (!seen.has(candidate.source_id)) {
+            db()
+              .prepare(
+                "UPDATE posts SET status='archived',updated_at=? WHERE id=?",
+              )
+              .run(timestamp, candidate.id);
+            writeItem.run(
+              id("import_item"),
+              migrationId,
+              "post",
+              candidate.source_id,
+              candidate.id,
+              "updated",
+              "source deleted; archived locally",
+              timestamp,
+            );
+            deleted++;
+          }
+      }
+      return {
+        created,
+        updated,
+        skipped,
+        conflicts,
+        deleted,
+        total: records.length,
+      };
+    });
+    const counts = write();
+    db()
+      .prepare(
+        "UPDATE wp_migrations SET status='completed',counts_json=?,checkpoint_json=?,deleted_count=?,conflict_count=?,completed_at=? WHERE id=?",
+      )
+      .run(
+        JSON.stringify({ ...counts, sourceChecksum: aggregateChecksum }),
+        JSON.stringify({
+          processed: records.length,
+          total: records.length,
+          completed: true,
+        }),
+        counts.deleted,
+        counts.conflicts,
+        now(),
+        migrationId,
+      );
+    audit("migration.applied", "migration", migrationId, null, counts);
+    return json(
+      { id: migrationId, mode, counts, sourceChecksum: aggregateChecksum },
+      201,
+    );
+  } catch (error) {
+    return json(
+      { error: error instanceof Error ? error.message : "migration failed" },
+      422,
+    );
+  }
+};

package/bundled-templates/astro-blog/starter/src/pages/api/ops/migrations/media.ts

@@ -0,0 +1,5 @@
+import type { APIRoute } from "astro";
+import { db, id, now } from "../../../../lib/db";
+import { json } from "../../../../lib/ops-auth";
+
+export const POST: APIRoute = async ({ request }) => { const body = await request.json().catch(() => null) as any; const items = Array.isArray(body?.items) ? body.items : []; if (!items.length) return json({ error: "items is required" }, 400); const timestamp = now(); let created = 0; const tx = db().transaction(() => { for (const item of items) { if (!item.sourceUrl) continue; db().prepare("INSERT INTO media_assets (id,source,source_id,source_url,local_url,mime_type,alt_text,caption,width,height,metadata_json,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source,source_id) DO UPDATE SET local_url=excluded.local_url,mime_type=excluded.mime_type,alt_text=excluded.alt_text,width=excluded.width,height=excluded.height,updated_at=excluded.updated_at").run(id("media"), "media-archive", String(item.sourceId || item.sourceUrl), item.sourceUrl, item.localUrl || null, item.mimeType || null, item.altText || "", item.caption || "", item.width || null, item.height || null, JSON.stringify({ archive: body.archive || null }), timestamp, timestamp); created++; } }); tx(); return json({ imported: created, status: "registered" }, 201); };