company-dossier 0.2.1 → 0.3.1

package/README.md

@@ -140,10 +140,69 @@ Tool input (same as the stdio server):
 { "target": "acme.com", "sections": ["overview", "tech", "risk"] }
 ```
 
+## Docs-site output
+
+`company-dossier` can turn a generated dossier into a themed, **static
+[Astro Starlight](https://starlight.astro.build) docs site** — one page per
+section, an autogenerated sidebar, and a "case file" (ink-on-paper) theme.
+
+```bash
+# scaffold a Starlight site under "<Company> DOSSIER/site/"
+company-dossier acme.com --out ./out --format site
+
+# then build it (Node-only):
+cd "./out/Acme DOSSIER/site" && npm install && npm run build   # → site/dist
+```
+
+Site-related flags:
+
+| Flag | Default | Meaning |
+|------|---------|---------|
+| `--format <folder\|site>` | `folder` | `site` scaffolds the Starlight project. |
+| `--deploy <none\|gh-pages>` | `none` | `gh-pages` installs, builds, and publishes `site/dist` via the [`gh-pages`](https://www.npmjs.com/package/gh-pages) package (an optional dep of the *generated* site). |
+| `--subdomain <host>` | — | Writes `public/CNAME` for a custom host. |
+| `--no-noindex` | (noindex on) | Opt out of the unlisted/noindex policy below. |
+
+### Unlisted by default (policy)
+
+Per-company dossier sites are **UNLISTED + NOINDEX by default**. Every generated
+page carries:
+
+- `<meta name="robots" content="noindex,nofollow">` (injected via Starlight `head`),
+- a visible disclaimer — *"Auto-generated from public sources — may be
+  inaccurate; not affiliated with the company"* — plus a takedown/contact line, and
+- a `public/robots.txt` with `Disallow: /` (and `public/.nojekyll`).
+
+Pass `--no-noindex` to allow indexing (robots meta is dropped and robots.txt
+becomes `Allow: /`).
+
+The scaffold (`<Company> DOSSIER/site/`) contains `package.json`
+(`astro` + `@astrojs/starlight`, optional `gh-pages`), `astro.config.mjs`,
+`src/content/config.ts` (loose docs schema), `src/content/docs/*.md`
+(one per section + an `index.md` hero from the overview),
+`src/styles/case-file.css`, and `public/` assets.
+
+### Library usage
+
+```ts
+import { buildDossier, writeDossier, generateSite } from 'company-dossier';
+
+const result = await buildDossier('acme.com');
+const folder = writeDossier(result, './out');           // "<Company> DOSSIER/"
+const site = await generateSite(result, folder, {       // → folder + "/site/"
+  noindex: true,        // default; set false to allow indexing
+  subdomain: 'acme.example.com', // optional → public/CNAME
+  title: 'Acme',        // optional override (defaults to company name)
+});
+console.log(site.siteDir, site.files.length);
+```
+
 ## Output
 
 - A `<Company> DOSSIER/` folder with `README.md`, nine numbered markdown
   sections, and `dossier.json`.
+- With `--format site`, additionally a `<Company> DOSSIER/site/` Astro
+  Starlight project (build it with `npm install && npm run build`).
 - With `--json`, the structured dossier is printed to stdout instead.
 
 ## Public sources only

package/dist/cli.js

@@ -20,6 +20,9 @@ function writeFileSafe(filePath, content) {
 function todayISO() {
   return (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
 }
+function slugify(name) {
+  return name.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/_+$/, "").replace(/^_+/, "");
+}
 function titleCase(str) {
   return str.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
 }
@@ -913,7 +916,7 @@ var SECTIONS = [
   "relationships",
   "risk"
 ];
-var GENERATOR = "company-dossier v0.1.0";
+var GENERATOR = "company-dossier v0.3.0";
 var HOMEPAGE = "https://companydossier.lol";
 var GAP = "_Gap: no public data found \u2014 requires manual research._";
 function deriveCompanyName(target, website) {
@@ -1405,19 +1408,304 @@ async function buildDossier(target, opts = {}) {
 }
 
 // src/index.ts
+import * as path3 from "path";
+
+// src/generators/site.ts
 import * as path2 from "path";
+var ASTRO_VERSION = "^5.7.0";
+var STARLIGHT_VERSION = "^0.34.0";
+var GH_PAGES_VERSION = "^6.3.0";
+var DISCLAIMER = "Auto-generated from public sources \u2014 may be inaccurate; not affiliated with the company.";
+var TAKEDOWN = "Takedown / contact: open an issue at https://github.com/ever-just/company-dossier/issues";
+function stripFrontmatter(content) {
+  if (content.startsWith("---\n")) {
+    const end = content.indexOf("\n---", 4);
+    if (end !== -1) {
+      const after = content.indexOf("\n", end + 1);
+      return after !== -1 ? content.slice(after + 1) : "";
+    }
+  }
+  return content;
+}
+function extractTitle2(content, fallback) {
+  const m = content.match(/^#\s+(.+?)\s*$/m);
+  if (!m) return fallback;
+  let t = m[1].trim();
+  t = t.replace(/^\d+\.\s*/, "");
+  t = t.replace(/\s*[—–-]\s*[^—–-]+$/, "");
+  return t.trim() || fallback;
+}
+function orderFromFilename(filePath) {
+  const m = path2.basename(filePath).match(/^(\d+)/);
+  return m ? parseInt(m[1], 10) : void 0;
+}
+function yamlString(value) {
+  return '"' + value.replace(/\\/g, "\\\\").replace(/"/g, '\\"') + '"';
+}
+function isOverview(filePath) {
+  const b = path2.basename(filePath).toLowerCase();
+  return b === "readme.md" || /^0*1[_.]/.test(b) || b.includes("overview");
+}
+function buildPackageJson(name, deploy) {
+  const pkg = {
+    name: `${slugify(name) || "company"}-dossier-site`,
+    private: true,
+    version: "0.0.1",
+    type: "module",
+    scripts: {
+      dev: "astro dev",
+      build: "astro build",
+      preview: "astro preview",
+      ...deploy ? { deploy: "gh-pages -d dist -t true" } : {}
+    },
+    dependencies: {
+      astro: ASTRO_VERSION,
+      "@astrojs/starlight": STARLIGHT_VERSION
+    },
+    // gh-pages is an optional dep of the GENERATED site, not the main package.
+    optionalDependencies: {
+      "gh-pages": GH_PAGES_VERSION
+    }
+  };
+  return JSON.stringify(pkg, null, 2) + "\n";
+}
+function buildAstroConfig(title, noindex) {
+  const headEntry = noindex ? `
+  head: [
+    {
+      tag: 'meta',
+      attrs: { name: 'robots', content: 'noindex,nofollow' },
+    },
+  ],` : "";
+  return `import { defineConfig } from 'astro/config';
+import starlight from '@astrojs/starlight';
+
+// Generated by company-dossier (--format site).
+// Per policy, per-company dossier sites are UNLISTED + NOINDEX by default.
+export default defineConfig({
+  integrations: [
+    starlight({
+      title: ${yamlString(title)},
+      description: ${yamlString(DISCLAIMER)},
+      tableOfContents: true,
+      customCss: ['./src/styles/case-file.css'],${headEntry}
+      // Sidebar autogenerated from the docs tree.
+      sidebar: [
+        {
+          label: 'Dossier',
+          autogenerate: { directory: '.' },
+        },
+      ],
+      // Footer disclaimer + takedown line on every page.
+      credits: false,
+      lastUpdated: false,
+      pagination: false,
+    }),
+  ],
+});
+`;
+}
+function buildContentConfig() {
+  return `import { defineCollection } from 'astro:content';
+import { docsLoader } from '@astrojs/starlight/loaders';
+import { docsSchema } from '@astrojs/starlight/schema';
+import { z } from 'astro:content';
+
+export const collections = {
+  docs: defineCollection({
+    loader: docsLoader(),
+    schema: docsSchema({
+      // Tolerant extra frontmatter keys for generated content.
+      extend: z.object({
+        order: z.number().optional(),
+        section: z.string().optional(),
+        sourceFile: z.string().optional(),
+        generatedAt: z.string().optional(),
+        company: z.string().optional(),
+        domain: z.string().optional(),
+        customNote: z.string().optional(),
+      }),
+    }),
+  }),
+};
+`;
+}
+function buildCaseFileCss() {
+  return `/* company-dossier "case file" theme.
+   Ink-on-paper light scheme expressed through Starlight design tokens. */
+
+:root {
+  /* Manila / paper accent (amber-brown ink stamp). */
+  --sl-color-accent-low: #efe6d2;
+  --sl-color-accent: #8a5a1f;
+  --sl-color-accent-high: #4d3210;
+
+  /* Ink on aged paper. */
+  --sl-color-white: #1c1a16;
+  --sl-color-gray-1: #2a2620;
+  --sl-color-gray-2: #423c31;
+  --sl-color-gray-3: #6b6253;
+  --sl-color-gray-4: #8c8270;
+  --sl-color-gray-5: #b9ad95;
+  --sl-color-gray-6: #e6dcc6;
+  --sl-color-gray-7: #f1e9d6;
+  --sl-color-black: #faf4e4;
+
+  --sl-color-bg: #faf4e4;
+  --sl-color-bg-nav: #f1e9d6;
+  --sl-color-bg-sidebar: #f4ecda;
+  --sl-color-text: #1c1a16;
+  --sl-color-text-accent: #8a5a1f;
+  --sl-color-hairline: #d8cbac;
+  --sl-color-hairline-light: #e6dcc6;
+
+  --sl-font: ui-serif, Georgia, 'Times New Roman', serif;
+  --sl-font-mono: ui-monospace, 'SFMono-Regular', 'Courier New', monospace;
+}
+
+/* Keep a dark variant readable too (folder-at-night). */
+:root[data-theme='dark'] {
+  --sl-color-accent-low: #2a2012;
+  --sl-color-accent: #d9a441;
+  --sl-color-accent-high: #f3d79a;
+  --sl-color-bg: #16140f;
+  --sl-color-bg-nav: #1d1a13;
+  --sl-color-bg-sidebar: #1b1812;
+  --sl-color-text: #ece2cc;
+  --sl-color-hairline: #36301f;
+}
+
+/* Markdown content reads like a typed report. */
+.sl-markdown-content {
+  line-height: 1.6;
+}
+.sl-markdown-content table {
+  font-family: var(--sl-font-mono);
+  font-size: 0.85rem;
+}
+
+/* CONFIDENTIAL / disclaimer banner injected at the top of every page. */
+.dossier-banner {
+  border: 1px dashed var(--sl-color-accent);
+  background: var(--sl-color-accent-low);
+  color: var(--sl-color-accent-high);
+  padding: 0.6rem 0.9rem;
+  margin: 0 0 1.25rem 0;
+  border-radius: 4px;
+  font-family: var(--sl-font-mono);
+  font-size: 0.8rem;
+  line-height: 1.45;
+}
+.dossier-banner strong {
+  display: inline-block;
+  letter-spacing: 0.18em;
+  text-transform: uppercase;
+  margin-right: 0.5rem;
+}
+.dossier-banner .dossier-takedown {
+  display: block;
+  margin-top: 0.35rem;
+  opacity: 0.85;
+}
+`;
+}
+function buildRobotsTxt(noindex) {
+  if (noindex) {
+    return "User-agent: *\nDisallow: /\n";
+  }
+  return "User-agent: *\nAllow: /\n";
+}
+function disclaimerMarkdown() {
+  return `:::caution[Disclaimer]
+**CONFIDENTIAL \u2014 UNVERIFIED.** ${DISCLAIMER}
+
+${TAKEDOWN}
+:::
+
+`;
+}
+function buildIndexDoc(result, files, title) {
+  const overviewFile = files.find((f) => path2.basename(f.path).toLowerCase() === "readme.md") || files.find((f) => isOverview(f.path));
+  const body = overviewFile ? stripFrontmatter(overviewFile.content) : "";
+  const bodyNoH1 = body.replace(/^#\s+.+?\r?\n/, "");
+  const frontmatter = [
+    "---",
+    `title: ${yamlString(title)}`,
+    "template: splash",
+    "hero:",
+    `  title: ${yamlString(title)}`,
+    `  tagline: ${yamlString("Intelligence dossier compiled from public sources.")}`,
+    "  actions:",
+    `    - text: Read the dossier`,
+    `      link: ./overview-identity/`,
+    `      icon: right-arrow`,
+    "---",
+    ""
+  ].join("\n");
+  return frontmatter + disclaimerMarkdown() + bodyNoH1.trim() + "\n";
+}
+function buildSectionDoc(file, title, order) {
+  const body = stripFrontmatter(file.content).replace(/^#\s+.+?\r?\n/, "");
+  const fm = ["---", `title: ${yamlString(title)}`];
+  if (order !== void 0) {
+    fm.push("sidebar:", `  order: ${order}`);
+  }
+  fm.push("---", "");
+  return fm.join("\n") + disclaimerMarkdown() + body.trim() + "\n";
+}
+async function generateSite(result, outDir, opts = {}) {
+  const noindex = opts.noindex !== false;
+  const title = (opts.title || result.meta.companyName || "Company Dossier").trim();
+  const siteDir = path2.join(outDir, "site");
+  const written = [];
+  const write = (rel, content) => {
+    const abs = path2.join(siteDir, rel);
+    writeFileSafe(abs, content);
+    written.push(abs);
+  };
+  const files = result.files;
+  write("package.json", buildPackageJson(title, true));
+  write("astro.config.mjs", buildAstroConfig(title, noindex));
+  write("src/content/config.ts", buildContentConfig());
+  write("src/styles/case-file.css", buildCaseFileCss());
+  write("tsconfig.json", JSON.stringify({ extends: "astro/tsconfigs/strict" }, null, 2) + "\n");
+  write("src/content/docs/index.md", buildIndexDoc(result, files, title));
+  const usedSlugs = /* @__PURE__ */ new Set(["index"]);
+  for (const file of files) {
+    const base = path2.basename(file.path).toLowerCase();
+    if (base === "dossier.json") continue;
+    if (base === "readme.md") continue;
+    if (!base.endsWith(".md")) continue;
+    const docTitle = extractTitle2(file.content, path2.basename(file.path, ".md"));
+    let slug = slugify(docTitle).replace(/_/g, "-") || slugify(base.replace(/\.md$/, ""));
+    let unique = slug;
+    let n = 2;
+    while (usedSlugs.has(unique)) unique = `${slug}-${n++}`;
+    usedSlugs.add(unique);
+    const order = orderFromFilename(file.path);
+    write(`src/content/docs/${unique}.md`, buildSectionDoc(file, docTitle, order));
+  }
+  write("public/.nojekyll", "");
+  write("public/robots.txt", buildRobotsTxt(noindex));
+  if (opts.subdomain && opts.subdomain.trim()) {
+    write("public/CNAME", opts.subdomain.trim() + "\n");
+  }
+  return { siteDir, files: written, noindex };
+}
+
+// src/index.ts
 function writeDossier(result, outDir) {
   const folderName = `${result.meta.companyName} DOSSIER`.trim();
   const safeFolder = folderName.replace(/[/\\]/g, "_");
-  const target = path2.join(outDir, safeFolder);
+  const target = path3.join(outDir, safeFolder);
   for (const file of result.files) {
-    writeFileSafe(path2.join(target, file.path), file.content);
+    writeFileSafe(path3.join(target, file.path), file.content);
   }
   return target;
 }
 
 // src/cli.ts
-var VERSION = "0.1.0";
+var VERSION = "0.3.0";
 var HELP = `company-dossier v${VERSION}
 Build a complete, sourced intelligence dossier on any company from public data.
 
@@ -1436,6 +1724,15 @@ OPTIONS
                         Available: ${SECTIONS.join(", ")}
   --max-pages <n>       Max internal pages to crawl (default: 25).
   --no-social-probe     Skip slow HEAD-probing of social platforms.
+  --format <fmt>        Output format: folder (default) or site.
+                        "site" scaffolds a themed, static Astro Starlight
+                        docs site under "<Company> DOSSIER/site/".
+  --deploy <target>     Deploy target for --format site: none (default) or
+                        gh-pages (builds site/dist and publishes via the
+                        gh-pages npm package).
+  --subdomain <host>    Custom host for the site; writes public/CNAME.
+  --no-noindex          Allow indexing. By POLICY, dossier sites are
+                        UNLISTED + NOINDEX by default; this opts out.
   --quiet               Suppress progress output.
   -h, --help            Show this help.
   -v, --version         Show version.
@@ -1450,6 +1747,12 @@ EXAMPLES
   company-dossier "Acme Corporation"
   company-dossier acme.com --json > acme.json
   company-dossier acme.com --sections overview,tech,risk
+  company-dossier acme.com --out ./out --format site
+  company-dossier acme.com --format site --deploy gh-pages --subdomain acme.example.com
+
+Generated dossier sites are UNLISTED + NOINDEX by default (noindex meta,
+robots.txt Disallow, and a visible "auto-generated / not affiliated"
+disclaimer on every page). Use --no-noindex to opt out.
 
 Public sources only \u2014 no private databases, no API keys required.
 Learn more: https://companydossier.lol
@@ -1460,6 +1763,9 @@ function parseArgs(argv) {
     json: false,
     quiet: false,
     skipSocialProbe: false,
+    format: "folder",
+    deploy: "none",
+    noindex: true,
     help: false,
     version: false
   };
@@ -1483,6 +1789,30 @@ function parseArgs(argv) {
       case "--no-social-probe":
         out.skipSocialProbe = true;
         break;
+      case "--no-noindex":
+        out.noindex = false;
+        break;
+      case "--format": {
+        const v = (argv[++i] ?? "").trim().toLowerCase();
+        if (v === "folder" || v === "site") {
+          out.format = v;
+        } else {
+          out.error = `Invalid --format: ${v || "(missing)"}. Use folder or site.`;
+        }
+        break;
+      }
+      case "--deploy": {
+        const v = (argv[++i] ?? "").trim().toLowerCase();
+        if (v === "none" || v === "gh-pages") {
+          out.deploy = v;
+        } else {
+          out.error = `Invalid --deploy: ${v || "(missing)"}. Use none or gh-pages.`;
+        }
+        break;
+      }
+      case "--subdomain":
+        out.subdomain = (argv[++i] ?? "").trim() || void 0;
+        break;
       case "--out":
         out.out = argv[++i] ?? out.out;
         break;
@@ -1565,11 +1895,65 @@ Run "company-dossier --help".
   log(`
 Dossier written to: ${folder}`);
   log(`Files: ${result.files.map((f) => f.path).join(", ")}`);
+  if (args.format === "site") {
+    log("\nScaffolding Astro Starlight docs site...");
+    const site = await generateSite(result, folder, {
+      subdomain: args.subdomain,
+      noindex: args.noindex
+    });
+    log(`Site scaffolded at: ${site.siteDir}`);
+    log(
+      `  ${site.files.length} files written` + (site.noindex ? " (UNLISTED + NOINDEX: robots meta + Disallow-all)" : " (indexable)")
+    );
+    log(`  Build it: cd "${site.siteDir}" && npm install && npm run build`);
+    if (args.deploy === "gh-pages") {
+      const code = await deployGhPages(site.siteDir, log);
+      if (code !== 0) return code;
+    } else {
+      log("  Deploy: skipped (--deploy none).");
+    }
+    if (!args.quiet) {
+      process.stdout.write(site.siteDir + "\n");
+    }
+    return 0;
+  }
   if (!args.quiet) {
     process.stdout.write(folder + "\n");
   }
   return 0;
 }
+async function deployGhPages(siteDir, log) {
+  const { spawn } = await import("child_process");
+  const npmCmd = process.platform === "win32" ? "npm.cmd" : "npm";
+  const run = (cmd, cmdArgs) => new Promise((resolve) => {
+    log(`  $ ${cmd} ${cmdArgs.join(" ")}`);
+    const child = spawn(cmd, cmdArgs, { cwd: siteDir, stdio: "inherit" });
+    child.on("error", (err) => {
+      process.stderr.write(`Deploy error: ${err.message}
+`);
+      resolve(1);
+    });
+    child.on("close", (c) => resolve(c ?? 0));
+  });
+  log("\nDeploying to GitHub Pages via gh-pages...");
+  let code = await run(npmCmd, ["install"]);
+  if (code !== 0) {
+    process.stderr.write("Deploy aborted: npm install failed in site/.\n");
+    return code;
+  }
+  code = await run(npmCmd, ["run", "build"]);
+  if (code !== 0) {
+    process.stderr.write("Deploy aborted: astro build failed.\n");
+    return code;
+  }
+  code = await run(npmCmd, ["run", "deploy"]);
+  if (code !== 0) {
+    process.stderr.write("Deploy aborted: gh-pages publish failed.\n");
+    return code;
+  }
+  log("Deployed site/dist to gh-pages branch.");
+  return 0;
+}
 main().then((code) => process.exit(code)).catch((err) => {
   process.stderr.write(`Fatal: ${err instanceof Error ? err.message : String(err)}
 `);

package/dist/index.d.ts

@@ -200,10 +200,42 @@ interface DossierResult {
  */
 declare function buildDossier(target: string, opts?: BuildOptions): Promise<DossierResult>;
 
+interface GenerateSiteOptions {
+    /** Custom subdomain/host. When set, writes a public/CNAME file. */
+    subdomain?: string;
+    /**
+     * Inject `<meta name="robots" content="noindex,nofollow">` on every page
+     * and a Disallow-all robots.txt. Defaults to TRUE (unlisted by policy).
+     */
+    noindex?: boolean;
+    /** Override the site title. Defaults to the dossier company name. */
+    title?: string;
+}
+interface GenerateSiteResult {
+    /** Absolute path to the generated `site/` directory. */
+    siteDir: string;
+    /** Absolute paths of every file written, relative to nothing — fully qualified. */
+    files: string[];
+    /** Whether noindex/unlisted policy was applied. */
+    noindex: boolean;
+}
+/**
+ * Turn a generated dossier into a themed, static Astro Starlight docs site.
+ *
+ * Scaffolds an Astro Starlight project under `<outDir>/site/`. Per policy,
+ * the site is UNLISTED + NOINDEX by default: every page carries a
+ * `noindex,nofollow` robots meta, a visible disclaimer, and a takedown line,
+ * and robots.txt disallows all crawling.
+ *
+ * Build it with `cd site && npm install && npm run build` (output in
+ * `site/dist`). Deploy is pluggable; pass a subdomain to emit a CNAME.
+ */
+declare function generateSite(result: DossierResult, outDir: string, opts?: GenerateSiteOptions): Promise<GenerateSiteResult>;
+
 /**
  * Write a built dossier to disk. Files are placed inside a
  * "<Company> DOSSIER/" folder under `outDir`. Returns the folder path.
  */
 declare function writeDossier(result: DossierResult, outDir: string): string;
 
-export { type BuildOptions, type DnsData, type DossierData, type DossierFile, type DossierMeta, type DossierResult, type PageData, type ProgressCallback, SECTIONS, type SearchData, type SectionId, type SocialProfile, type TechStackData, type USASpendingContract, type WaybackData, type WebsiteData, buildDossier, collectDns, collectSearch, collectWayback, collectWebsite, extractTechStack, slugify, writeDossier };
+export { type BuildOptions, type DnsData, type DossierData, type DossierFile, type DossierMeta, type DossierResult, type GenerateSiteOptions, type GenerateSiteResult, type PageData, type ProgressCallback, SECTIONS, type SearchData, type SectionId, type SocialProfile, type TechStackData, type USASpendingContract, type WaybackData, type WebsiteData, buildDossier, collectDns, collectSearch, collectWayback, collectWebsite, extractTechStack, generateSite, slugify, writeDossier };