@tikoci/rosetta 0.6.7 → 0.6.8

package/src/extract-all-versions.ts

@@ -2,10 +2,14 @@
 /**
  * extract-all-versions.ts — Extract command trees from all RouterOS versions.
  *
- * Discovers RouterOS versions from restraml and extracts each inspect.json
- * (preferring extra/ variant) into command_versions.
+ * Discovers RouterOS versions from restraml and extracts each version's
+ * command tree into schema_nodes/command_versions.
  *
- * The latest stable version is loaded as the primary commands table.
+ * Prefers deep-inspect.{x86,arm64}.json (multi-arch, completion data) when
+ * available; falls back to inspect.json (legacy) for older versions.
+ *
+ * The latest stable version is loaded as the primary (rebuilds schema_nodes
+ * and commands tables). All other versions only add to the junction tables.
  *
  * Usage:
  *   bun run src/extract-all-versions.ts [restraml-base-url-or-local-docs-dir]
@@ -20,8 +24,15 @@ const SOURCE = process.argv[2];
 interface VersionInfo {
   version: string;
   channel: "stable" | "development";
+  /** deep-inspect.x86.json path/URL (null if not available) */
+  deepX86: string | null;
+  /** deep-inspect.arm64.json path/URL (null if not available) */
+  deepArm64: string | null;
+  /** Legacy inspect.json path/URL (fallback when no deep-inspect) */
   inspectPath: string;
   hasExtra: boolean;
+  /** true when at least one deep-inspect file is available */
+  hasDeepInspect: boolean;
 }
 
 function classifyChannel(version: string): "stable" | "development" {
@@ -55,18 +66,36 @@ async function discoverRemoteVersions(): Promise<VersionInfo[]> {
   const versionNames = await discoverRemoteVersionList();
   const baseUrl = RESTRAML_PAGES_URL;
 
-  // restraml publishes extra/ for every version that has extra-packages,
-  // and the GitHub API listing already confirmed these dirs exist.
-  // Assume extra/inspect.json is available (restraml always generates it for CHR builds).
-  return versionNames
-    .filter((name) => /^\d+\.\d+/.test(name))
-    .map((name) => ({
+  // For each version, check if deep-inspect files exist (HEAD probe).
+  // Deep-inspect files are at: <baseUrl>/<version>/extra/deep-inspect.{x86,arm64}.json
+  // Fall back to: <baseUrl>/<version>/extra/inspect.json
+  const results: VersionInfo[] = [];
+
+  for (const name of versionNames.filter((n) => /^\d+\.\d+/.test(n))) {
+    const x86Url = `${baseUrl}/${name}/extra/deep-inspect.x86.json`;
+    const arm64Url = `${baseUrl}/${name}/extra/deep-inspect.arm64.json`;
+    const inspectUrl = `${baseUrl}/${name}/extra/inspect.json`;
+
+    // Probe deep-inspect availability via HEAD request (fast, no body)
+    const [x86Ok, arm64Ok] = await Promise.all([
+      fetch(x86Url, { method: "HEAD" }).then((r) => r.ok).catch(() => false),
+      fetch(arm64Url, { method: "HEAD" }).then((r) => r.ok).catch(() => false),
+    ]);
+
+    const hasDeepInspect = x86Ok || arm64Ok;
+
+    results.push({
       version: name,
       channel: classifyChannel(name),
-      inspectPath: `${baseUrl}/${name}/extra/inspect.json`,
+      deepX86: x86Ok ? x86Url : null,
+      deepArm64: arm64Ok ? arm64Url : null,
+      inspectPath: inspectUrl,
       hasExtra: true,
-    }))
-    .sort((a, b) => compareVersions(a.version, b.version));
+      hasDeepInspect,
+    });
+  }
+
+  return results.sort((a, b) => compareVersions(a.version, b.version));
 }
 
 function discoverLocalVersions(docsDir: string): VersionInfo[] {
@@ -74,18 +103,26 @@ function discoverLocalVersions(docsDir: string): VersionInfo[] {
   return entries
     .map((name) => {
       const dir = resolve(docsDir, name);
+      const deepX86Path = resolve(dir, "extra/deep-inspect.x86.json");
+      const deepArm64Path = resolve(dir, "extra/deep-inspect.arm64.json");
       const extraPath = resolve(dir, "extra/inspect.json");
       const basePath = resolve(dir, "inspect.json");
       const hasExtra = existsSync(extraPath);
       const inspectPath = hasExtra ? extraPath : basePath;
+      const deepX86 = existsSync(deepX86Path) ? deepX86Path : null;
+      const deepArm64 = existsSync(deepArm64Path) ? deepArm64Path : null;
+      const hasDeepInspect = deepX86 !== null || deepArm64 !== null;
 
-      if (!existsSync(inspectPath)) return null;
+      if (!existsSync(inspectPath) && !hasDeepInspect) return null;
 
       return {
         version: name,
         channel: classifyChannel(name),
+        deepX86,
+        deepArm64,
         inspectPath,
         hasExtra,
+        hasDeepInspect,
       };
     })
     .filter((v): v is VersionInfo => v !== null)
@@ -112,10 +149,16 @@ const latest = latestStable || versions[versions.length - 1];
 console.log(`Latest stable: ${latest?.version ?? "none"}`);
 
 // Run extraction for each version
-// First pass: accumulate all non-primary versions
-// Second pass: primary version (replaces commands table)
+// For versions with deep-inspect: use extract-schema.ts (multi-arch, completion data)
+// For versions without: fall back to extract-commands.ts (legacy)
+// Primary version (latest stable) rebuilds the main tables; others accumulate only.
 
-const extractCmd = resolve(import.meta.dir, "extract-commands.ts");
+const extractSchemaCmd = resolve(import.meta.dir, "extract-schema.ts");
+const extractCommandsCmd = resolve(import.meta.dir, "extract-commands.ts");
+
+const deepCount = versions.filter((v) => v.hasDeepInspect).length;
+const legacyCount = versions.length - deepCount;
+console.log(`Deep-inspect versions: ${deepCount}, legacy inspect.json: ${legacyCount}`);
 
 let extracted = 0;
 for (const v of versions) {
@@ -128,12 +171,31 @@ for (const v of versions) {
   ];
 
   console.log(`\n${"=".repeat(60)}`);
-  console.log(`${isPrimary ? "PRIMARY" : "accumulate"}: ${v.version} (${v.channel})`);
 
-  const proc = Bun.spawnSync(["bun", "run", extractCmd, v.inspectPath, ...flags], {
-    cwd: resolve(import.meta.dir, ".."),
-    stdio: ["inherit", "inherit", "inherit"],
-  });
+  let proc: ReturnType<typeof Bun.spawnSync>;
+
+  if (v.hasDeepInspect) {
+    // Use extract-schema.ts with both arch files
+    const schemaFlags = [
+      ...(v.deepX86 ? [`--x86=${v.deepX86}`] : []),
+      ...(v.deepArm64 ? [`--arm64=${v.deepArm64}`] : []),
+      ...flags,
+    ];
+    console.log(`${isPrimary ? "PRIMARY" : "accumulate"}: ${v.version} (${v.channel}) [deep-inspect]`);
+
+    proc = Bun.spawnSync(["bun", "run", extractSchemaCmd, ...schemaFlags], {
+      cwd: resolve(import.meta.dir, ".."),
+      stdio: ["inherit", "inherit", "inherit"],
+    });
+  } else {
+    // Legacy fallback: use extract-commands.ts
+    console.log(`${isPrimary ? "PRIMARY" : "accumulate"}: ${v.version} (${v.channel}) [legacy inspect.json]`);
+
+    proc = Bun.spawnSync(["bun", "run", extractCommandsCmd, v.inspectPath, ...flags], {
+      cwd: resolve(import.meta.dir, ".."),
+      stdio: ["inherit", "inherit", "inherit"],
+    });
+  }
 
   if (proc.exitCode !== 0) {
     console.error(`FAILED: ${v.version} (exit ${proc.exitCode})`);