dogsbay 0.2.0-beta.2 → 0.2.0-beta.21

package/dist/index.js

@@ -15,6 +15,7 @@ import { siteInit } from "./commands/site-init.js";
 import { siteBuild } from "./commands/site-build.js";
 import { siteCheck } from "./commands/site-check.js";
 import { siteDev, sitePreview } from "./commands/site-dev.js";
+import { agentInstall } from "./commands/agent.js";
 // Read version from the runtime package.json so `dogsbay --version`
 // never drifts from what's published. Walks one level up from
 // `dist/index.js` to `package.json` (works in both monorepo dev and
@@ -62,7 +63,7 @@ site
     .option("--edit-uri <path>", "Repo path prefix for edit links (default: blob/main/content/)")
     .option("--copyright <text>", "Footer copyright text")
     .option("--theme <name>", "Theme preset (default | material)")
-    .option("--deploy <target>", "Deploy target (cloudflare-workers)")
+    .option("--deploy <target>", "Deploy target (cloudflare-workers | github-pages)")
     .option("--content <path>", "Path to source markdown (relative to config)")
     .option("--from <format>", "Source format (auto | dogsbay-md | mkdocs | obsidian | starlight | mdx)")
     .option("--nav <path>", "Path to explicit nav file (.json/.yml)")
@@ -96,7 +97,7 @@ site
     .option("--edit-uri <path>", "Override site.editUri")
     .option("--copyright <text>", "Override site.copyright")
     .option("--theme <name>", "Override theme (default | material)")
-    .option("--deploy <target>", "Override deploy.target (cloudflare-workers)")
+    .option("--deploy <target>", "Override deploy.target (cloudflare-workers | github-pages)")
     .option("--content <path>", "Override content.source")
     .option("--from <format>", "Override content.from")
     .option("--nav <path>", "Override content.nav")
@@ -108,8 +109,12 @@ site
     .option("--ai-input <yes|no>", "Override Content-Signal aiInput")
     .option("--ai-search <yes|no>", "Override Content-Signal search")
     .option("--include-drafts", "Include status: draft pages (default: excluded from prod build)")
-    .option("--publish", "Build the full publish matrix (every source, ignoring primary: flag). " +
-    "Default builds only sources marked primary: true (or all sources when no primary is set).")
+    .option("--primary-only", "Build only sources marked primary: true (skip non-primary). " +
+    "Default builds the full publish matrix — every locale / version / namespace " +
+    "the writer declared. Use --primary-only for fast single-source iteration in CI " +
+    "lint jobs (rare).")
+    .option("--publish", "Deprecated — `dogsbay site build` defaults to publish mode now. " +
+    "Kept as a no-op for compatibility with older scripts.")
     .option("--refresh", "Wipe the per-source git cache before resolving — forces re-clone of every git: source. " +
     "Useful when a tracked branch's HEAD has moved.")
     .action((dir, options) => siteBuild(dir, options));
@@ -218,6 +223,16 @@ program
     .option("--concurrency <n>", "Maximum concurrent fetches (default: 3)", "3")
     .option("--rate-limit <ms>", "Minimum ms between request batches (default: 200)", "200")
     .action((url, options) => pull(url, options));
+// ── `dogsbay agent` — wire skill discovery for LLM agents ──────────────
+const agent = program
+    .command("agent")
+    .description("Wire Dogsbay platform skills into LLM-agent discovery paths");
+agent
+    .command("install")
+    .description("Install platform skills + per-agent discovery symlinks")
+    .option("--agent <names>", "Comma-separated list (e.g. claude,cursor)")
+    .option("--all", "Install for every supported agent")
+    .action((options) => agentInstall(undefined, options));
 program
     .command("export-techdocs")
     .description("Post-process Astro build output into Backstage TechDocs format")

package/dist/passthrough-astro.js

@@ -0,0 +1,152 @@
+/**
+ * Passthrough Astro page collection.
+ *
+ * Hand-authored `.astro` files that live under a content source's
+ * directory get copied verbatim to `src/pages/<basePath>/...` at
+ * build time. Authors opt in by listing the file in `nav.yml`; the
+ * intersection of "files on disk" and "hrefs in nav" defines the
+ * passthrough set.
+ *
+ * See plans/passthrough-astro-pages.md.
+ */
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join, posix, relative, sep } from "node:path";
+/**
+ * Walk `contentDir` for `.astro` files, compute each one's canonical
+ * href, and return only the entries whose href appears in the
+ * resolved nav. Files outside the nav are ignored — passthrough is
+ * opt-in to avoid accidentally publishing scratch components.
+ */
+export function collectPassthroughEntries(contentDir, nav, options) {
+    if (!existsSync(contentDir))
+        return [];
+    const navHrefs = collectNavHrefs(nav);
+    const candidates = walkAstroFiles(contentDir, contentDir);
+    const entries = [];
+    for (const sourceAbs of candidates) {
+        const source = toPosix(relative(contentDir, sourceAbs));
+        const href = sourceToHref(source, options.basePath);
+        if (!navHrefs.has(href))
+            continue;
+        entries.push({
+            source,
+            sourceAbs,
+            outputRelPath: sourceToOutputRelPath(source, options.basePath),
+            href,
+        });
+    }
+    return entries;
+}
+/**
+ * Recursively gather every `.astro` file under `dir`. Skips
+ * `node_modules`, dot-directories, and any directory named
+ * `_components` / `_partials` (a common convention for "this is
+ * shared, don't publish it" — leaves authors an obvious escape
+ * hatch when they want to ship private helpers alongside content).
+ */
+function walkAstroFiles(dir, root) {
+    const out = [];
+    let entries = [];
+    try {
+        entries = readdirSync(dir, { withFileTypes: true });
+    }
+    catch {
+        return out;
+    }
+    for (const entry of entries) {
+        if (entry.name.startsWith("."))
+            continue;
+        if (entry.name === "node_modules")
+            continue;
+        if (entry.name === "_components" || entry.name === "_partials")
+            continue;
+        const full = join(dir, entry.name);
+        if (entry.isDirectory()) {
+            out.push(...walkAstroFiles(full, root));
+        }
+        else if (entry.isFile() && entry.name.endsWith(".astro")) {
+            out.push(full);
+        }
+    }
+    return out;
+}
+/**
+ * Compute the public-facing href for a source path. Mirrors the
+ * slug logic in `format-dogsbay-md/src/nav-file.ts:fileToHref` so a
+ * passthrough .astro file produces the same href as a hypothetical
+ * .md sibling at the same path.
+ *
+ * - "tutorials/playground.astro" → "/docs/tutorials/playground"
+ * - "reference/index.astro"      → "/docs/reference"
+ * - "index.astro"                → "/docs"
+ */
+function sourceToHref(source, basePath) {
+    let slug = source.replace(/\.astro$/i, "");
+    if (slug.endsWith("/index"))
+        slug = slug.slice(0, -"/index".length);
+    if (slug === "index")
+        slug = "";
+    const prefix = basePath.endsWith("/") ? basePath.slice(0, -1) : basePath;
+    return slug ? `${prefix}/${slug}` : prefix || "";
+}
+/**
+ * Compute the output path (relative to outputDir) where a passthrough
+ * source should be copied. Preserves the source's directory shape
+ * under `src/pages/<basePath segments>/`. Index files survive as
+ * `index.astro` so Astro's directory-routing matches the nav href.
+ */
+function sourceToOutputRelPath(source, basePath) {
+    const baseSegs = basePath.split("/").filter((s) => s.length > 0);
+    const sourceSegs = source.split("/");
+    return ["src", "pages", ...baseSegs, ...sourceSegs].join(sep);
+}
+/** Walk a NavItem tree collecting every href into a Set. */
+function collectNavHrefs(nav) {
+    const out = new Set();
+    const stack = [...nav];
+    while (stack.length > 0) {
+        const item = stack.pop();
+        if (item.href)
+            out.add(item.href);
+        if (item.children)
+            stack.push(...item.children);
+    }
+    return out;
+}
+/**
+ * Normalize backslashes (Windows path separators) to forward slashes
+ * before doing any href / slug computation. The slug shape is
+ * URL-flavoured even on Windows.
+ */
+function toPosix(p) {
+    return p.split(sep).join(posix.sep);
+}
+/**
+ * Build the slug set covered by the passthrough entries. Used by
+ * site-build to assert no collision with generated slugs from
+ * `emitAstroPages`.
+ */
+export function passthroughSlugs(entries) {
+    return new Set(entries.map((e) => e.href));
+}
+/**
+ * Convenience guard — verify every passthrough source still exists
+ * on disk. Walking already only returns existing files, but this
+ * helper is useful when entries are constructed externally (e.g. in
+ * tests).
+ */
+export function assertPassthroughFilesExist(entries) {
+    for (const entry of entries) {
+        if (!existsSync(entry.sourceAbs)) {
+            throw new Error(`Passthrough Astro source missing: ${entry.source} ` +
+                `(resolved: ${entry.sourceAbs})`);
+        }
+    }
+    // Touch statSync to detect non-file entries (defensive)
+    for (const entry of entries) {
+        const st = statSync(entry.sourceAbs);
+        if (!st.isFile()) {
+            throw new Error(`Passthrough Astro source is not a file: ${entry.source}`);
+        }
+    }
+}

package/dist/registry.js

@@ -388,6 +388,14 @@ export const registry = {
         primitives: [],
         description: "Clickable card with link",
     },
+    icon: {
+        name: "icon",
+        files: ["Icon.astro", "index.ts"],
+        dependencies: ["@dogsbay/icons"],
+        registryDependencies: [],
+        primitives: [],
+        description: "Build-time-resolved icon (Lucide default; mdi:, simple-icons:, etc. via Iconify)",
+    },
     sidebar: {
         name: "sidebar",
         files: ["Sidebar.astro", "SidebarContent.astro", "SidebarGroup.astro", "SidebarGroupContent.astro", "SidebarGroupLabel.astro", "SidebarHeader.astro", "SidebarInset.astro", "SidebarMenu.astro", "SidebarMenuButton.astro", "SidebarMenuItem.astro", "SidebarNavTree.astro", "SidebarProvider.astro", "SidebarRail.astro", "SidebarSeparator.astro", "SidebarTrigger.astro", "sidebar.ts", "index.ts"],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dogsbay",
-  "version": "0.2.0-beta.2",
+  "version": "0.2.0-beta.21",
   "description": "CLI for Dogsbay — scaffold, build, and serve documentation sites with markdown / MkDocs / Obsidian / OpenAPI sources",
   "type": "module",
   "bin": {
@@ -9,6 +9,7 @@
   "files": [
     "dist",
     "bin",
+    "skills",
     "README.md"
   ],
   "keywords": [
@@ -25,19 +26,20 @@
   ],
   "dependencies": {
     "cheerio": "^1.0.0",
+    "chokidar": "^4.0.0",
     "commander": "^13.0.0",
     "markdown-it": "^14.0.0",
     "picocolors": "^1.1.0",
     "prompts": "^2.4.2",
     "yaml": "^2.8.3",
-    "@dogsbay/format-mkdocs": "0.2.0-beta.2",
-    "@dogsbay/format-astro": "0.2.0-beta.2",
-    "@dogsbay/format-mdx": "0.2.0-beta.2",
-    "@dogsbay/format-obsidian": "0.2.0-beta.2",
-    "@dogsbay/format-starlight": "0.2.0-beta.2",
-    "@dogsbay/format-dogsbay-md": "0.2.0-beta.2",
-    "@dogsbay/types": "0.2.0-beta.2",
-    "@dogsbay/format-openapi": "0.2.0-beta.2"
+    "@dogsbay/format-mkdocs": "0.2.0-beta.21",
+    "@dogsbay/format-astro": "0.2.0-beta.21",
+    "@dogsbay/format-obsidian": "0.2.0-beta.21",
+    "@dogsbay/format-mdx": "0.2.0-beta.21",
+    "@dogsbay/format-starlight": "0.2.0-beta.21",
+    "@dogsbay/format-dogsbay-md": "0.2.0-beta.21",
+    "@dogsbay/format-openapi": "0.2.0-beta.21",
+    "@dogsbay/types": "0.2.0-beta.21"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

package/skills/platform/agent-readiness/SKILL.md

@@ -0,0 +1,262 @@
+---
+name: dogsbay:agent-readiness
+description: How Dogsbay sites expose content to LLM agents and search indexers — llms.txt, llms-full.txt, .md mirrors, Content-Signal HTTP headers, robots.txt. Use when configuring agent.* in dogsbay.config.yml or debugging agent consumption.
+---
+
+# Agent readiness
+
+Every Dogsbay site is built to be **agent-readable by default**.
+Three mechanisms work together so that any modern LLM, search
+engine, or AI-answer-engine can consume the docs as cleanly as
+a human reader:
+
+1. **llms.txt** at the root — the canonical agent index
+2. **`.md` mirror** for every page — the prose body without
+   chrome
+3. **Content-Signal HTTP headers** — IETF-track signal for "what
+   AI use is permitted"
+
+All three are emitted at `dogsbay site build` time. Toggleable
+via the `agent:` block in `dogsbay.config.yml`.
+
+## llms.txt
+
+The standard at [llmstxt.org](https://llmstxt.org/) — a single
+file at the site root that lists every page with title +
+description + URL. Two flavours:
+
+- `/llms.txt` — short index (title + description + URL per page,
+  grouped by section). The agent's "table of contents."
+- `/llms-full.txt` — full index with the markdown body of every
+  page concatenated. The agent's "everything in one paste."
+
+Per-section mini-indexes also emit (`/llms-${section}.txt` for
+each top-level nav group), so an agent can pull just the
+relevant slice without grabbing the whole site.
+
+Format example (`/llms.txt`):
+
+```
+# Acme Docs
+
+> Documentation for the Acme platform.
+
+## Getting started
+
+- [Installation](/docs/install): Install the CLI on macOS, Linux, or Windows.
+- [Quickstart](/docs/quickstart): Your first request in 60 seconds.
+
+## API reference
+
+- [List pets](/docs/api/pets/list-pets): Returns paginated list.
+- [Create a pet](/docs/api/pets/create-pet): Idempotent creation.
+```
+
+Toggleable:
+
+```yaml
+agent:
+  llmsTxt: true              # default true; set false to omit
+```
+
+## `.md` mirror
+
+Every emitted page has a sibling `.md` route that returns the
+markdown source (or a faithful prose rendering of it) with
+`Content-Type: text/markdown`.
+
+For a page at `/docs/api/pets/list-pets`, the mirror is at
+`/docs/api/pets/list-pets.md`. For `/docs/`, it's at `/docs.md`.
+
+Why two URLs? A human visiting `/docs/api/pets/list-pets` gets
+the rich HTML page with components, sidebar, search. An agent
+hitting `/docs/api/pets/list-pets.md` gets just the prose —
+faster to parse, no HTML noise, no dependency on a Cloudflare
+worker for content negotiation.
+
+### Discovery via `<link rel="alternate">`
+
+Every HTML page emits:
+
+```html
+<link rel="alternate" type="text/markdown" href="/docs/api/pets/list-pets.md">
+```
+
+Agents that follow `rel="alternate"` find the mirror without
+guessing at URL conventions. Anthropic's prompt-cache, Mintlify's
+agents, and several others do this.
+
+Toggleable:
+
+```yaml
+agent:
+  mdMirror: true             # default true
+```
+
+### Per-page opt-out
+
+Some pages don't have useful prose mirrors (e.g. landing pages
+that are mostly hero components). Opt out per-page:
+
+```yaml
+---
+title: Home
+mdMirror: false
+---
+```
+
+Or via the global `agent.mdMirror: false`.
+
+### Content negotiation (Cloudflare worker)
+
+Astro's static-mode output doesn't pass per-request headers to
+middleware, so the in-build middleware can't respond to
+`Accept: text/markdown` by serving the `.md` body. The current
+mitigation: the explicit `.md` URL is always available, and
+`<link rel="alternate">` exposes it. A Cloudflare worker that
+does proper content negotiation at the edge is planned (see
+`plans/cloudflare-deploy-content-negotiation.md`).
+
+## Content-Signal HTTP headers
+
+Per the IETF Content-Signal draft, sites can declare AI-use
+permissions via HTTP headers:
+
+```
+Content-Signal: aiTrain=no, aiInput=yes, search=yes
+```
+
+Three keys:
+
+| Key | Values | Meaning |
+|---|---|---|
+| `aiTrain` | `yes` / `no` | May this content be used for AI model training? |
+| `aiInput` | `yes` / `no` | May this content be used as input to a live AI session (RAG, prompt context)? |
+| `search` | `yes` / `no` | May this content be indexed by search engines? |
+
+Configure via `agent.contentSignal`:
+
+```yaml
+agent:
+  contentSignal:
+    aiTrain: "no"            # don't use my docs to train models
+    aiInput: "yes"           # but DO use them as live context (e.g. for users in Claude / Cursor)
+    search: "yes"            # standard search indexing OK
+```
+
+Emitted in two places:
+
+- `public/_headers` — Cloudflare Pages / Vercel / Netlify pick
+  this up at the edge automatically
+- `<meta>` tags in HTML head — for hosts that don't read
+  `_headers`
+
+## robots.txt
+
+Auto-emitted at `public/robots.txt` based on `noindex` settings
++ Content-Signal `search` value. Disallows crawlers when
+`search: "no"`; otherwise allows everything.
+
+For per-page `noindex`, the `robots` meta tag handles it (see
+`dogsbay:frontmatter-fields`).
+
+## Per-page LLM action UI
+
+Beyond the data side, Dogsbay can render an action cluster
+("Copy as markdown", "Open in Claude", "Open in ChatGPT") on
+each page:
+
+```yaml
+agent:
+  llmsTxt: true
+  mdMirror: true
+
+llmActions:
+  enabled: true
+  providers: [claude, chatgpt, perplexity, gemini]   # render order
+  placement: header                                   # header | inline | both
+  copyButton: true
+  promptTemplate: "Read this docs page: {url}"
+  footerLink: true
+```
+
+`{url}` resolves to the absolute `.md` mirror URL. The user
+clicks "Open in Claude" → goes to `claude.ai/new?q=...` with a
+prepopulated prompt that pulls the markdown into Claude's
+context.
+
+Per-page opt-out via `llmActions: false` in frontmatter.
+
+## What agents see
+
+When an LLM is given the URL of a Dogsbay site:
+
+1. It fetches `/llms.txt` (table of contents)
+2. Picks pages relevant to the question
+3. Fetches each as `/{path}.md` (full prose)
+4. Reads `Content-Signal` to know if it's allowed to use the
+   content as context (typically yes if `aiInput=yes`)
+
+That's a self-contained agent-consumption loop with no special
+configuration on the agent's side.
+
+## Common patterns
+
+### Public docs, no AI training, allow live context
+
+```yaml
+agent:
+  llmsTxt: true
+  mdMirror: true
+  contentSignal:
+    aiTrain: "no"
+    aiInput: "yes"
+    search: "yes"
+```
+
+The default for most teams. Their docs help users in AI sessions
+but don't end up in training data.
+
+### Internal docs (no public agent access)
+
+```yaml
+agent:
+  llmsTxt: false             # don't advertise to crawlers
+  mdMirror: true             # but keep the dev-side .md surface
+  contentSignal:
+    aiTrain: "no"
+    aiInput: "no"
+    search: "no"
+```
+
+Plus host-side auth (Cloudflare Access, Vercel password, etc.)
+to gate the site itself.
+
+### Marketing-site mode (everything open)
+
+```yaml
+agent:
+  llmsTxt: true
+  mdMirror: true
+  contentSignal:
+    aiTrain: "yes"           # put us in the training data; we want the visibility
+    aiInput: "yes"
+    search: "yes"
+```
+
+## Common mistakes
+
+- ❌ Setting `agent.mdMirror: false` and expecting llms.txt to
+  still link to .md files — the index emits whatever URLs the
+  build produces. If mirrors aren't built, the index can't link
+  to them.
+- ❌ Trusting `Accept: text/markdown` content negotiation today —
+  static-mode middleware doesn't see request headers. Use the
+  explicit `.md` URL.
+- ❌ `aiTrain: "no"` + a public-internet-readable site —
+  Content-Signal is **declarative**, not enforceable. Crawlers
+  can ignore it. For real protection, gate access at the
+  network level.
+- ❌ Mistyping the Content-Signal values (`"true"` instead of
+  `"yes"`) — the loader doesn't normalise; the header emits
+  literally what you wrote.