@dastyaresocial/ds-cs 0.0.0-stage → 0.1.0

package/src/registry/sources.mjs

@@ -0,0 +1,395 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+import { CliError } from "../utils/errors.mjs";
+import { isDirectory } from "../utils/fs.mjs";
+import { extractBuffer, extractTarGz } from "./archive.mjs";
+import { cachedBuffer, fetchBuffer, fetchJson, fetchText } from "./fetch.mjs";
+
+const execFileAsync = promisify(execFile);
+
+/** The first-party registry. Skills ship from the DS-CS repo's `skills/` dir. */
+export const DS_CS_REPO = "dastyare-social/DS-CS";
+export const DS_CS_REF = "main";
+export const DS_CS_SKILLS_DIR = "skills";
+
+export const SOURCE_TYPES = {
+  DS_CS: "ds-cs",
+  GITHUB: "github",
+  GIT: "git",
+  URL: "url",
+  LOCAL: "local",
+};
+
+function hashKey(value) {
+  return crypto.createHash("sha256").update(value).digest("hex").slice(0, 16);
+}
+
+/** Relative when that is readable, absolute when it would be a chain of `../`. */
+function readable(target) {
+  const relative = path.relative(process.cwd(), target);
+  return relative === "" || relative.startsWith("..") ? target : relative;
+}
+
+async function makeTempDir() {
+  return fs.mkdtemp(path.join(os.tmpdir(), "ds-cs-src-"));
+}
+
+/* -------------------------------------------------------------------------- */
+/* First-party registry                                                       */
+/* -------------------------------------------------------------------------- */
+
+/**
+ * Where the registry can be found inside this package.
+ *
+ * Two layouts have to work. In a DS-CS checkout the CLI lives at
+ * `scripts/cli/src/registry/`, so the registry is the repo root's `skills/`.
+ * Published to npm, `sync-skills` copies the registry to `scripts/cli/skills/`
+ * so it travels inside the package. Both are checked against the package root,
+ * cheapest first.
+ */
+function bundledRegistryRoots() {
+  // `../..` from src/registry/ is the package root in both layouts.
+  const packageRoot = fileURLToPath(new URL("../..", import.meta.url));
+  return [
+    path.join(packageRoot, "skills"), // published layout: <pkg>/skills
+    path.resolve(packageRoot, "..", "..", "skills"), // checkout: <repo>/skills
+  ];
+}
+
+/** Walk up from `cwd` looking for a DS-CS checkout that ships a `skills/` dir. */
+async function findLocalCheckout(cwd) {
+  let directory = path.resolve(cwd);
+  for (let depth = 0; depth < 6; depth += 1) {
+    const skills = path.join(directory, DS_CS_SKILLS_DIR);
+    if (await isDirectory(skills)) return directory;
+    const parent = path.dirname(directory);
+    if (parent === directory) break;
+    directory = parent;
+  }
+  return null;
+}
+
+/**
+ * Fetch only the `skills/` subtree of the DS-CS repository.
+ *
+ * The full repo tarball is ~28MB — almost all of it `public/` and generated
+ * assets — to read a handful of markdown files. Listing the tree once and
+ * pulling just the registry's files keeps the fallback under a megabyte, and
+ * keeps it working under the archive size limit.
+ */
+async function fetchRegistrySubtree(ref) {
+  const listing = await fetchJson(
+    `https://api.github.com/repos/${DS_CS_REPO}/git/trees/${ref}?recursive=1`
+  );
+
+  if (!Array.isArray(listing.tree)) {
+    throw new CliError(`GitHub returned no tree for ${DS_CS_REPO}@${ref}`, {
+      hint: "The branch may not exist, or the API rate limit may be reached.",
+    });
+  }
+
+  const wanted = listing.tree.filter(
+    (node) =>
+      node.type === "blob" &&
+      node.path.startsWith(`${DS_CS_SKILLS_DIR}/`) &&
+      // Skip build artefacts that would otherwise ride along in the registry.
+      !node.path.endsWith("/.DS_Store") &&
+      !path.posix.basename(node.path).startsWith("._")
+  );
+
+  if (wanted.length === 0) {
+    throw new CliError(`No ${DS_CS_SKILLS_DIR}/ directory at ${DS_CS_REPO}@${ref}`);
+  }
+
+  const extraction = await makeTempDir();
+  for (const node of wanted) {
+    const target = path.join(extraction, node.path);
+    await fs.mkdir(path.dirname(target), { recursive: true });
+    const body = await fetchText(
+      `https://raw.githubusercontent.com/${DS_CS_REPO}/${ref}/${node.path}`
+    );
+    await fs.writeFile(target, body);
+  }
+
+  return { root: extraction, fileCount: wanted.length };
+}
+
+/**
+ * Resolve the DS-CS registry to a directory of skills.
+ *
+ * Order matters, and each step exists for a reason:
+ *
+ *  1. A local checkout — so editing a skill in the repo and reinstalling it
+ *     works, instead of resolving against whatever `main` says.
+ *  2. The copy bundled in this package — works offline, matches the CLI
+ *     version the user actually has, and needs no network at all.
+ *  3. GitHub — a last resort for a CLI older than the skills it wants.
+ */
+export async function resolveDsCs({ cwd, preferLocal = true }) {
+  if (preferLocal) {
+    const localRoot = await findLocalCheckout(cwd);
+    if (localRoot) {
+      return {
+        id: DS_CS_REPO,
+        type: SOURCE_TYPES.DS_CS,
+        root: localRoot,
+        origin: "local",
+        describe: () => `${DS_CS_REPO} (local checkout at ${readable(localRoot)})`,
+      };
+    }
+  }
+
+  for (const bundled of bundledRegistryRoots()) {
+    if (await isDirectory(bundled)) {
+      return {
+        id: DS_CS_REPO,
+        type: SOURCE_TYPES.DS_CS,
+        root: path.resolve(bundled, ".."),
+        origin: "bundled",
+        describe: () => `${DS_CS_REPO} (bundled with this CLI)`,
+      };
+    }
+  }
+
+  const { root, fileCount } = await fetchRegistrySubtree(DS_CS_REF);
+  return {
+    id: DS_CS_REPO,
+    type: SOURCE_TYPES.DS_CS,
+    root,
+    origin: "remote",
+    describe: () => `${DS_CS_REPO}@${DS_CS_REF} (${fileCount} files)`,
+  };
+}
+
+/* -------------------------------------------------------------------------- */
+/* GitHub                                                                     */
+/* -------------------------------------------------------------------------- */
+
+export function parseGithubShorthand(spec) {
+  const match = /^([a-zA-Z0-9_.-]+)\/([a-zA-Z0-9_.-]+)$/.exec(spec);
+  return match ? { owner: match[1], repo: match[2], subpath: null, ref: null } : null;
+}
+
+export function parseGithubUrl(spec) {
+  let url;
+  try {
+    url = new URL(spec);
+  } catch {
+    return null;
+  }
+  if (url.hostname !== "github.com" && url.hostname !== "www.github.com") return null;
+
+  const segments = url.pathname.replace(/^\/+|\/+$/g, "").split("/").filter(Boolean);
+  if (segments.length < 2) return null;
+
+  const [owner, repo] = segments;
+  let ref = null;
+  let subpath = null;
+
+  if (segments[2] === "tree" && segments[3]) {
+    ref = segments[3];
+    subpath = segments.slice(4).join("/") || null;
+  }
+
+  return { owner, repo, subpath, ref };
+}
+
+/**
+ * Fetch a GitHub repository as a tarball.
+ *
+ * Deliberately not `git clone`: a tarball is one HTTP request, needs no git
+ * binary, and cannot run a repository's hooks. Branches and tags both resolve
+ * through `refs/heads` / `refs/tags`.
+ */
+export async function resolveGithub({ owner, repo, ref, subpath }, spec) {
+  const resolvedRef = ref ?? "main";
+  const headRef = resolvedRef.includes("/") ? `heads/${resolvedRef}` : `heads/${resolvedRef}`;
+  const url = `https://codeload.github.com/${owner}/${repo}/tar.gz/refs/${headRef}`;
+
+  let buffer;
+  try {
+    buffer = await cachedBuffer(url, hashKey(url));
+  } catch (error) {
+    throw new CliError(`Could not fetch ${owner}/${repo}@${resolvedRef} from GitHub`, {
+      hint: "Check the branch exists, or set GITHUB_TOKEN for a private repository.",
+      cause: error,
+    });
+  }
+
+  const extraction = await makeTempDir();
+  await extractTarGz(buffer, extraction, { strip: 1 });
+
+  const root = subpath ? path.join(extraction, subpath) : extraction;
+  if (!(await isDirectory(root))) {
+    throw new CliError(`${owner}/${repo} has no "${subpath}" directory`, {
+      hint: "Check the path in the /tree/<ref>/<path> URL.",
+    });
+  }
+
+  return {
+    id: spec ?? `${owner}/${repo}`,
+    type: SOURCE_TYPES.GITHUB,
+    root,
+    origin: "remote",
+    describe: () => `${owner}/${repo}@${resolvedRef}${subpath ? `/${subpath}` : ""}`,
+  };
+}
+
+/* -------------------------------------------------------------------------- */
+/* Git                                                                        */
+/* -------------------------------------------------------------------------- */
+
+export function isGitUrl(spec) {
+  return /^(git@|git:\/\/|ssh:\/\/|git\+)/.test(spec) || /^https?:\/\//.test(spec);
+}
+
+/** Clone is only reached for non-GitHub git hosts, where no tarball API exists. */
+export async function resolveGit(spec) {
+  const staging = await makeTempDir();
+  try {
+    await execFileAsync("git", [
+      "clone",
+      "--depth",
+      "1",
+      "--quiet",
+      spec,
+      path.join(staging, "repo"),
+    ]);
+  } catch (error) {
+    throw new CliError(`git clone failed for ${spec}`, {
+      hint: "Git must be installed, and credentials configured for private remotes.",
+      cause: error,
+    });
+  }
+  return {
+    id: spec,
+    type: SOURCE_TYPES.GIT,
+    root: path.join(staging, "repo"),
+    origin: "remote",
+    describe: () => spec,
+  };
+}
+
+/* -------------------------------------------------------------------------- */
+/* Direct URLs                                                                */
+/* -------------------------------------------------------------------------- */
+
+const ARCHIVE_EXTENSIONS = [".zip", ".tar", ".tar.gz", ".tgz"];
+
+function looksLikeArchive(url) {
+  const pathName = new URL(url).pathname.toLowerCase();
+  return ARCHIVE_EXTENSIONS.some((extension) => pathName.endsWith(extension));
+}
+
+/**
+ * A URL is either a single SKILL.md or an archive of a skill tree. The body is
+ * sniffed rather than trusted: a `.md` URL that returns HTML is an error page,
+ * and treating it as a skill would install a file with no frontmatter.
+ */
+export async function resolveUrl(spec) {
+  const buffer = await fetchBuffer(spec);
+  const isArchive = looksLikeArchive(spec);
+  const isBinary =
+    buffer.includes(0x50) && buffer.includes(0x4b) && buffer[0] === 0x50 && buffer[1] === 0x4b;
+
+  if (isArchive || isBinary) {
+    const staging = await makeTempDir();
+    const filename = new URL(spec).pathname.split("/").pop() || "archive.zip";
+    await extractBuffer(buffer, filename, staging, { strip: 1 });
+    return {
+      id: spec,
+      type: SOURCE_TYPES.URL,
+      root: staging,
+      origin: "remote",
+      describe: () => spec,
+    };
+  }
+
+  const body = buffer.toString("utf8");
+  if (!/^---\r?\n/.test(body)) {
+    throw new CliError(`${spec} does not look like a SKILL.md or a skills archive`, {
+      hint: "The URL must serve a SKILL.md, or a .zip / .tar.gz archive of one.",
+    });
+  }
+
+  const skillDir = await makeTempDir();
+  await fs.mkdir(skillDir, { recursive: true });
+  await fs.writeFile(path.join(skillDir, "SKILL.md"), body);
+  return {
+    id: spec,
+    type: SOURCE_TYPES.URL,
+    root: skillDir,
+    origin: "remote",
+    describe: () => spec,
+  };
+}
+
+/* -------------------------------------------------------------------------- */
+/* Local                                                                      */
+/* -------------------------------------------------------------------------- */
+
+export async function resolveLocal(spec) {
+  const expanded = spec.startsWith("~") ? path.join(os.homedir(), spec.slice(1)) : spec;
+  const root = path.resolve(process.cwd(), expanded);
+  if (!(await isDirectory(root))) {
+    throw new CliError(`Local skill path does not exist: ${root}`);
+  }
+  return {
+    id: spec,
+    type: SOURCE_TYPES.LOCAL,
+    root,
+    origin: "local",
+    describe: () => readable(root),
+  };
+}
+
+/* -------------------------------------------------------------------------- */
+/* Dispatch                                                                   */
+/* -------------------------------------------------------------------------- */
+
+export function isLocalPath(spec) {
+  return (
+    spec.startsWith(".") ||
+    spec.startsWith("/") ||
+    spec.startsWith("~") ||
+    spec.startsWith("file://")
+  );
+}
+
+/**
+ * Turn a source spec into a materialised directory of skills.
+ *
+ * An empty spec means the DS-CS registry, which is what makes the common case
+ * (`ds-cs add brand-config`) work with no ceremony.
+ */
+export async function resolveSource(spec, { cwd = process.cwd(), preferLocal = true } = {}) {
+  const trimmed = (spec ?? "").trim();
+
+  if (trimmed === "" || trimmed === "ds-cs" || trimmed === "@ds-cs") {
+    return resolveDsCs({ cwd, preferLocal });
+  }
+
+  if (isLocalPath(trimmed)) {
+    return resolveLocal(trimmed.replace(/^file:\/\//, ""));
+  }
+
+  const github = trimmed.startsWith("http")
+    ? parseGithubUrl(trimmed)
+    : parseGithubShorthand(trimmed);
+
+  if (github) {
+    return resolveGithub(github, trimmed.startsWith("http") ? trimmed : trimmed);
+  }
+
+  if (isGitUrl(trimmed)) {
+    return resolveGit(trimmed);
+  }
+
+  return resolveUrl(trimmed);
+}

package/src/skills/discover.mjs

@@ -0,0 +1,170 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+import { CliError } from "../utils/errors.mjs";
+import { hashDirectory, isDirectory, isFile } from "../utils/fs.mjs";
+import { readSkillManifest } from "./frontmatter.mjs";
+
+/**
+ * Directories that hold skills in a source tree. A `skills/` at the repo root is
+ * the canonical first-party layout for DS-CS; the rest are recognised so a
+ * third-party repo can be consumed by the same command.
+ */
+export const CONTAINER_DIRECTORIES = [
+  "skills",
+  "skills/.curated",
+  "skills/.experimental",
+  "skills/.system",
+  ".agents/skills",
+  ".claude/skills",
+  "agent/skills",
+  "data/skills",
+];
+
+/** How deep to walk inside a container. 3 covers skills/<name>/SKILL.md plus two category levels. */
+const MAX_DEPTH = 3;
+
+async function collectSkillFiles(dir, depth, out) {
+  if (depth > MAX_DEPTH) return out;
+
+  let entries;
+  try {
+    entries = await import("node:fs/promises").then((fs) =>
+      fs.readdir(dir, { withFileTypes: true })
+    );
+  } catch {
+    return out;
+  }
+
+  const hasSkillFile = entries.some(
+    (entry) => entry.isFile() && entry.name === "SKILL.md"
+  );
+
+  if (hasSkillFile) {
+    out.push(path.join(dir, "SKILL.md"));
+    // A skill directory is a unit; never descend into it looking for nested
+    // skills (reference docs and scripts live there).
+    return out;
+  }
+
+  for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+    if (!entry.isDirectory()) continue;
+    if (entry.name.startsWith(".") && entry.name !== ".curated" && entry.name !== ".system") {
+      continue;
+    }
+    await collectSkillFiles(path.join(dir, entry.name), depth + 1, out);
+  }
+
+  return out;
+}
+
+/**
+ * Find every skill in a directory tree.
+ *
+ * Returns descriptors carrying the skill's directory *and* the container-relative
+ * path (`skillPath` in the lockfile), because `ds-cs update` needs the latter to
+ * re-fetch a single skill from a source it has never fully cloned.
+ */
+export async function discoverSkills(root) {
+  const skills = [];
+  const seen = new Set();
+
+  // A repository whose root *is* the skill is valid — register it before the
+  // containers so a root skill is never missed.
+  const rootSkillFile = path.join(root, "SKILL.md");
+  if (await isFile(rootSkillFile)) {
+    seen.add(rootSkillFile);
+    const manifest = await readSkillManifest(rootSkillFile);
+    skills.push({
+      name: manifest.name,
+      description: manifest.description,
+      internal: manifest.internal,
+      dir: root,
+      skillFile: rootSkillFile,
+      skillPath: "SKILL.md",
+      hash: await hashDirectory(root),
+    });
+  }
+
+  for (const container of CONTAINER_DIRECTORIES) {
+    const containerDir = path.join(root, container);
+    if (!(await isDirectory(containerDir))) continue;
+
+    const files = await collectSkillFiles(containerDir, 1, []);
+    for (const file of files) {
+      if (seen.has(file)) continue;
+      seen.add(file);
+
+      const dir = path.dirname(file);
+      const manifest = await readSkillManifest(file);
+      skills.push({
+        name: manifest.name,
+        description: manifest.description,
+        internal: manifest.internal,
+        dir,
+        skillFile: file,
+        skillPath: path.relative(root, file).split(path.sep).join("/"),
+        hash: await hashDirectory(dir),
+      });
+    }
+  }
+
+  return skills.sort((a, b) => a.name.localeCompare(b.name));
+}
+
+export function assertSkillExists(skills, name) {
+  const found = skills.find((skill) => skill.name === name);
+  if (found) return found;
+
+  const available = skills.filter((skill) => !skill.internal).map((skill) => skill.name);
+  throw new CliError(`No skill named "${name}" in that source`, {
+    hint:
+      available.length > 0
+        ? `Available: ${available.join(", ")}`
+        : "That source contains no skills.",
+  });
+}
+
+/**
+ * List skills already installed into an agent directory.
+ *
+ * `discoverSkills` is for *source* trees, where skills sit inside a `skills/`
+ * container. An installed directory is already that container, so its skills are
+ * one level down — running discovery against it would search for containers
+ * inside itself and find nothing.
+ */
+export async function listInstalledSkills(skillsDirectory) {
+  if (!(await isDirectory(skillsDirectory))) return [];
+
+  const entries = await fs.readdir(skillsDirectory, { withFileTypes: true });
+  const skills = [];
+
+  for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+    if (!entry.isDirectory() && !entry.isSymbolicLink()) continue;
+
+    const dir = path.join(skillsDirectory, entry.name);
+    const skillFile = path.join(dir, "SKILL.md");
+    if (!(await isFile(skillFile))) continue;
+
+    let manifest;
+    try {
+      manifest = await readSkillManifest(skillFile);
+    } catch {
+      // A directory that is not a skill (a stray file, an unrelated folder) is
+      // not an error — it just is not a skill.
+      continue;
+    }
+
+    skills.push({
+      name: manifest.name,
+      description: manifest.description,
+      internal: manifest.internal,
+      dir,
+      skillFile,
+      skillPath: `${entry.name}/SKILL.md`,
+      hash: await hashDirectory(dir),
+    });
+  }
+
+  return skills;
+}

package/src/skills/frontmatter.mjs

@@ -0,0 +1,85 @@
+/**
+ * Parse the YAML frontmatter of a SKILL.md.
+ *
+ * Deliberately not a general YAML parser: the Agent Skills spec only needs
+ * `name`, `description` and `metadata.internal` out of the header, and pulling a
+ * YAML dependency into a zero-dependency CLI for three scalar keys is not a
+ * trade worth making. Anything more complex is ignored rather than misread.
+ */
+
+function parseScalar(raw) {
+  const value = raw.trim();
+  if (
+    (value.startsWith('"') && value.endsWith('"')) ||
+    (value.startsWith("'") && value.endsWith("'"))
+  ) {
+    return value.slice(1, -1);
+  }
+  if (value === "true") return true;
+  if (value === "false") return false;
+  return value;
+}
+
+export function parseFrontmatter(source) {
+  const match = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?/.exec(source);
+  if (!match) return {};
+
+  const result = {};
+  let currentKey = null;
+
+  for (const line of match[1].split(/\r?\n/)) {
+    if (line.trim() === "" || line.trim().startsWith("#")) continue;
+
+    // A nested key (`metadata:` block) only records the key; its indented
+    // children are read on the following iterations.
+    const indented = /^\s+/.test(line);
+    const separator = line.indexOf(":");
+
+    if (separator === -1) continue;
+    const key = line.slice(0, separator).trim();
+    const rest = line.slice(separator + 1);
+
+    if (indented && currentKey === "metadata") {
+      if (key === "internal") result.internal = parseScalar(rest);
+      continue;
+    }
+
+    if (rest.trim() === "") {
+      currentKey = key;
+      continue;
+    }
+
+    currentKey = null;
+    result[key] = parseScalar(rest);
+  }
+
+  return result;
+}
+
+export function hasFrontmatter(source) {
+  return /^---\r?\n/.test(source);
+}
+
+/**
+ * Read a skill's identity. Throws with an actionable message when the header is
+ * missing, because a nameless skill is unaddressable — `ds-cs remove <name>`
+ * could never find it again.
+ */
+export async function readSkillManifest(skillFile) {
+  const fs = await import("node:fs/promises");
+  const source = await fs.readFile(skillFile, "utf8");
+  const frontmatter = parseFrontmatter(source);
+
+  if (!frontmatter.name || !frontmatter.description) {
+    throw new Error(
+      `${skillFile} is missing required frontmatter ("name" and "description" are both required by the Agent Skills spec)`
+    );
+  }
+
+  return {
+    name: String(frontmatter.name).trim(),
+    description: String(frontmatter.description).trim(),
+    internal: frontmatter.internal === true || frontmatter.internal === "true",
+    body: source,
+  };
+}

package/src/utils/errors.mjs

@@ -0,0 +1,13 @@
+/**
+ * An error the CLI knows how to explain to a human. `hint` is printed under
+ * the message; anything without a hint is probably a bug.
+ */
+export class CliError extends Error {
+  constructor(message, options = {}) {
+    super(message);
+    this.name = "CliError";
+    this.hint = options.hint;
+    this.exitCode = options.exitCode ?? 1;
+    if (options.cause) this.cause = options.cause;
+  }
+}