@navneet_25/tempjs 1.0.2 → 1.0.3

package/README.md

@@ -141,14 +141,13 @@ tempjs hotel --remote
 
 ## How fetching works
 
-`tempjs hotel` does **not** clone the entire repository.
+`tempjs hotel` does **not** run `git clone` and does **not** call the GitHub REST API per file.
 
-1. The CLI reads the GitHub API tree for the configured branch.
-2. It filters files under `templates/hotel-website-template/`.
-3. Only those files are downloaded (via blob API or raw content).
-4. Files are validated in a temporary directory, then copied to your project.
+1. The CLI downloads **one** repository archive from `codeload.github.com` (a single HTTP request — no API rate-limit issues for public repos).
+2. It extracts only `templates/hotel-website-template/` from that archive into a temp directory.
+3. Files are validated, then copied into your project directory.
 
-Other templates are not transferred.
+The archive contains the whole templates repo on the wire, but only the requested template folder is extracted locally. No `GITHUB_TOKEN` is required for public repositories.
 
 ## Repository structure
 

package/cli/fetch.js

@@ -1,231 +1,189 @@
-import { createWriteStream } from "node:fs";
-import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
+import { execFileSync } from "node:child_process";
+import { createWriteStream, existsSync } from "node:fs";
+import { mkdir, mkdtemp, readdir, readFile, rm, stat } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { dirname, join } from "node:path";
+import { join } from "node:path";
 import { pipeline } from "node:stream/promises";
 import { Readable } from "node:stream";
 
-const GITHUB_API = "https://api.github.com";
-const RAW_BASE = "https://raw.githubusercontent.com";
-const LARGE_FILE_BYTES = 1024 * 1024;
+const CODELOAD_BASE = "https://codeload.github.com";
 
 /**
- * Download a single template directory from GitHub without cloning the full repo.
+ * Download a single template directory from GitHub without cloning or per-file API calls.
+ *
+ * Strategy: one tarball download from codeload.github.com (not the REST API), then
+ * extract only templates/<templateDirectory>/ from the archive. This uses a single
+ * HTTP request instead of 60+ blob API calls, avoiding unauthenticated rate limits.
+ *
  * @param {import('./config.js').RepositoryConfig} repo
  * @param {string} templateDirectory
  * @returns {Promise<string>} Path to temp directory containing template files
  */
 export async function fetchTemplateFromGitHub(repo, templateDirectory) {
-  const prefix = `${repo.templatesPath}/${templateDirectory}/`;
-  const tree = await fetchRepoTree(repo);
-
-  const blobs = tree
-    .filter((entry) => entry.type === "blob" && entry.path.startsWith(prefix))
-    .map((entry) => ({
-      githubPath: entry.path,
-      relPath: entry.path.slice(prefix.length),
-      sha: entry.sha,
-    }));
-
-  if (blobs.length === 0) {
-    throw new Error(
-      `Template directory not found on GitHub: ${repo.templatesPath}/${templateDirectory}`
-    );
-  }
-
   const tempRoot = await mkdtemp(join(tmpdir(), "template-cli-"));
+  const tarballPath = join(tempRoot, "archive.tar.gz");
   const templateRoot = join(tempRoot, "template");
-  await mkdir(templateRoot, { recursive: true });
-
-  const concurrency = 8;
-  let index = 0;
-  const errors = [];
-
-  async function worker() {
-    while (index < blobs.length) {
-      const current = index++;
-      const blob = blobs[current];
-      try {
-        await downloadBlob(repo, blob, templateRoot);
-      } catch (error) {
-        errors.push({ relPath: blob.relPath, error });
-      }
-    }
-  }
 
-  await Promise.all(Array.from({ length: concurrency }, () => worker()));
-
-  if (errors.length > 0) {
-    const detail = errors
-      .slice(0, 5)
-      .map((e) => `${e.relPath}: ${e.error.message}`)
-      .join("\n");
-    throw new Error(
-      `Failed to download ${errors.length} file(s).\n${detail}`
+  try {
+    await downloadTarball(repo, tarballPath);
+    await extractTemplateFromTarball(
+      tarballPath,
+      templateRoot,
+      repo,
+      templateDirectory
     );
+    await readFile(join(templateRoot, "package.json"));
+    return templateRoot;
+  } catch (error) {
+    await rm(tempRoot, { recursive: true, force: true });
+    throw error;
   }
-
-  return templateRoot;
 }
 
 /**
  * @param {import('./config.js').RepositoryConfig} repo
- * @returns {Promise<Array<{ path: string, type: string, sha: string }>>}
+ * @param {string} destPath
  */
-async function fetchRepoTree(repo) {
-  const refUrl = `${GITHUB_API}/repos/${repo.owner}/${repo.repo}/git/ref/heads/${repo.branch}`;
-  const refResponse = await githubRequest(refUrl);
-
-  if (!refResponse.ok) {
-    throw await formatGitHubError(
-      refResponse,
-      `Could not resolve branch "${repo.branch}" for ${repo.owner}/${repo.repo}`
-    );
-  }
-
-  const refData = await refResponse.json();
-  const commitSha = refData.object?.sha;
-  if (!commitSha) {
-    throw new Error(`Invalid ref response for branch ${repo.branch}`);
-  }
-
-  const commitUrl = `${GITHUB_API}/repos/${repo.owner}/${repo.repo}/git/commits/${commitSha}`;
-  const commitResponse = await githubRequest(commitUrl);
-  if (!commitResponse.ok) {
-    throw await formatGitHubError(commitResponse, "Could not fetch commit metadata");
-  }
+async function downloadTarball(repo, destPath) {
+  const branch = encodeURIComponent(repo.branch);
+  const url = `${CODELOAD_BASE}/${repo.owner}/${repo.repo}/tar.gz/${branch}`;
 
-  const commitData = await commitResponse.json();
-  const treeSha = commitData.tree?.sha;
-  if (!treeSha) {
-    throw new Error("Invalid commit response: missing tree SHA");
-  }
+  const response = await fetch(url, {
+    headers: buildHeaders(),
+    redirect: "follow",
+  });
 
-  const treeUrl = `${GITHUB_API}/repos/${repo.owner}/${repo.repo}/git/trees/${treeSha}?recursive=1`;
-  const treeResponse = await githubRequest(treeUrl);
-  if (!treeResponse.ok) {
-    throw await formatGitHubError(treeResponse, "Could not fetch repository tree");
+  if (!response.ok) {
+    let detail = `Could not download ${repo.owner}/${repo.repo}@${repo.branch}`;
+    if (response.status === 404) {
+      detail += "\nVerify the repository, branch, and templates.json settings.";
+    }
+    if (response.status === 401 || response.status === 403) {
+      detail += "\nFor private repositories, set GITHUB_TOKEN or GH_TOKEN.";
+    }
+    throw new Error(`${detail} (HTTP ${response.status})`);
   }
 
-  const treeData = await treeResponse.json();
-  if (treeData.truncated) {
-    throw new Error(
-      "Repository tree is too large for a single API request. Contact the repository maintainer."
-    );
+  if (!response.body) {
+    throw new Error("Empty response while downloading repository archive");
   }
 
-  return treeData.tree ?? [];
+  await pipeline(Readable.fromWeb(response.body), createWriteStream(destPath));
 }
 
 /**
+ * @param {string} tarballPath
+ * @param {string} destDir
  * @param {import('./config.js').RepositoryConfig} repo
- * @param {{ githubPath: string, relPath: string, sha: string }} blob
- * @param {string} templateRoot
+ * @param {string} templateDirectory
  */
-async function downloadBlob(repo, blob, templateRoot) {
-  const { githubPath, relPath, sha } = blob;
+async function extractTemplateFromTarball(
+  tarballPath,
+  destDir,
+  repo,
+  templateDirectory
+) {
+  const listing = execFileSync("tar", ["-tzf", tarballPath], {
+    encoding: "utf8",
+    maxBuffer: 64 * 1024 * 1024,
+  });
 
-  if (relPath.includes(".git/") || relPath === ".git" || githubPath.includes("/.git/")) {
-    return;
+  const lines = listing.split("\n").filter(Boolean);
+  if (lines.length === 0) {
+    throw new Error("Repository archive is empty");
   }
 
-  const baseName = relPath.includes("/") ? relPath.slice(relPath.lastIndexOf("/") + 1) : relPath;
-  if (baseName === ".env" || (baseName.startsWith(".env.") && baseName !== ".env.example")) {
-    return;
-  }
+  const archiveRoot = lines[0].replace(/\/$/, "").split("/")[0];
+  const templateArchivePath = `${archiveRoot}/${repo.templatesPath}/${templateDirectory}`;
+  const templatePrefix = `${templateArchivePath}/`;
 
-  const targetPath = join(templateRoot, relPath);
-  await mkdir(dirname(targetPath), { recursive: true });
+  const hasTemplate = lines.some(
+    (line) => line === templatePrefix || line.startsWith(templatePrefix)
+  );
 
-  const blobUrl = `${GITHUB_API}/repos/${repo.owner}/${repo.repo}/git/blobs/${sha}`;
-  const blobResponse = await githubRequest(blobUrl);
-  if (!blobResponse.ok) {
-    throw await formatGitHubError(blobResponse, `Failed to fetch blob for ${relPath}`);
+  if (!hasTemplate) {
+    throw new Error(
+      `Template directory not found in repository: ${repo.templatesPath}/${templateDirectory}`
+    );
   }
 
-  const blobData = await blobResponse.json();
+  const stripComponents = templateArchivePath.split("/").length;
 
-  if (blobData.encoding === "base64" && blobData.content && blobData.size <= LARGE_FILE_BYTES) {
-    const content = Buffer.from(blobData.content.replace(/\n/g, ""), "base64");
-    await writeFile(targetPath, content);
-    return;
-  }
+  await mkdir(destDir, { recursive: true });
+
+  execFileSync(
+    "tar",
+    [
+      "-xzf",
+      tarballPath,
+      "-C",
+      destDir,
+      `--strip-components=${stripComponents}`,
+      templateArchivePath,
+    ],
+    { stdio: "pipe", maxBuffer: 64 * 1024 * 1024 }
+  );
 
-  await downloadRawFile(repo, githubPath, targetPath);
+  await removeSkippedFiles(destDir);
 }
 
 /**
- * @param {import('./config.js').RepositoryConfig} repo
- * @param {string} githubPath
- * @param {string} targetPath
+ * Remove files that must never appear in generated projects.
+ * @param {string} dir
  */
-async function downloadRawFile(repo, githubPath, targetPath) {
-  const url = `${RAW_BASE}/${repo.owner}/${repo.repo}/${repo.branch}/${githubPath}`;
-  const response = await fetch(url, {
-    headers: buildHeaders(),
-    redirect: "follow",
-  });
-
-  if (!response.ok) {
-    throw new Error(`HTTP ${response.status} for ${url}`);
-  }
+async function removeSkippedFiles(dir) {
+  if (!existsSync(dir)) return;
+
+  const entries = await readDirSafe(dir);
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory) {
+      if (entry.name === ".git") {
+        await rm(fullPath, { recursive: true, force: true });
+        continue;
+      }
+      await removeSkippedFiles(fullPath);
+      continue;
+    }
 
-  if (!response.body) {
-    throw new Error(`Empty response body for ${url}`);
+    if (shouldSkipFileName(entry.name)) {
+      await rm(fullPath, { force: true });
+    }
   }
+}
 
-  await pipeline(Readable.fromWeb(response.body), createWriteStream(targetPath));
+/**
+ * @param {string} dir
+ */
+async function readDirSafe(dir) {
+  const names = await readdir(dir);
+  const result = [];
+  for (const name of names) {
+    const fullPath = join(dir, name);
+    const info = await stat(fullPath);
+    result.push({ name, isDirectory: info.isDirectory() });
+  }
+  return result;
 }
 
 /**
- * @param {string} url
+ * @param {string} name
  */
-async function githubRequest(url) {
-  return fetch(url, {
-    headers: {
-      ...buildHeaders(),
-      Accept: "application/vnd.github+json",
-      "X-GitHub-Api-Version": "2022-11-28",
-    },
-  });
+function shouldSkipFileName(name) {
+  if (name === ".env" || (name.startsWith(".env.") && name !== ".env.example")) {
+    return true;
+  }
+  return false;
 }
 
 function buildHeaders() {
   const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
   if (token) {
-    return { Authorization: `Bearer ${token}` };
+    return { Authorization: `token ${token}` };
   }
   return {};
 }
 
-/**
- * @param {Response} response
- * @param {string} message
- */
-async function formatGitHubError(response, message) {
-  let detail = message;
-  try {
-    const body = await response.json();
-    if (body?.message) detail = `${message}: ${body.message}`;
-  } catch {
-    // ignore parse errors
-  }
-
-  if (response.status === 404) {
-    return new Error(
-      `${detail}\nVerify TEMPLATES_REPO_URL / templates.json repository settings.`
-    );
-  }
-
-  if (response.status === 403) {
-    return new Error(
-      `${detail}\nGitHub API rate limit may apply. Set GITHUB_TOKEN for higher limits.`
-    );
-  }
-
-  return new Error(`${detail} (HTTP ${response.status})`);
-}
-
 /**
  * @param {string} packageRoot
  * @param {string} templatesPath

package/cli/index.js

@@ -39,7 +39,7 @@ ENVIRONMENT
   TEMPLATES_REPO_REPO      GitHub repository name
   TEMPLATES_REPO_BRANCH    Branch name (default: main)
   TEMPLATE_USE_REMOTE=1    Always fetch from GitHub
-  GITHUB_TOKEN / GH_TOKEN    GitHub token for API rate limits
+  GITHUB_TOKEN / GH_TOKEN    Optional — required only for private repositories
 
 CONFIGURATION
   Repository and template mappings live in templates.json at the package root.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@navneet_25/tempjs",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "CLI to instantiate website project templates from a single GitHub repository",
   "type": "module",
   "bin": {