@canmi/me 2026.1004.3 → 2026.1004.5

package/README.md

@@ -1,7 +1,7 @@
 # @canmi/me
 
 The author's own facts, for the author's own programs: their sites' addresses and the world's
-(`@canmi/me/urls`), who they are (`@canmi/me/identity`), and the languages they write in
-(`@canmi/me/locales`). The `canmi` crate is the same addresses and identity for Rust.
+(`@canmi/me/urls`), who they are (`@canmi/me/identity`), the languages they write in (`@canmi/me/locales`), and what every
+public host of theirs says to a crawler and an agent (`@canmi/me/robots`). The `canmi` crate is the same addresses and identity for Rust.
 
 Versioned by the day it was released, `YYYY.MDD.N`; see the repository's `spec/repository.md`.

package/dist/robots/src/index.d.ts

@@ -0,0 +1,105 @@
+//#region robots/src/index.d.ts
+/** The opening every policy shares. */
+export declare const robotsTxtBase: readonly ["# https://www.robotstxt.org/robotstxt.html", 'User-agent: *'];
+/**
+ * How a page's content may be used, said once and written in both spellings: Cloudflare's
+ * `Content-Signal` and the IETF AI Preferences draft's `Content-Usage`. Only a host that serves
+ * pages says it; a store of bytes or an API has rules and nothing more.
+ */
+export declare const SIGNALS: {
+  readonly search: true;
+  readonly aiInput: true;
+  readonly aiTrain: true;
+};
+/** Each spelling as a name and a value: a robots.txt line, and a header on a page or markdown. */
+export declare const SIGNAL_HEADERS: readonly [readonly ['Content-Signal', `search=${string}, ai-input=${string}, ai-train=${string}`], readonly ['Content-Usage', `search=${string}, ai-use=${string}, train-ai=${string}`]];
+export declare const signalLines: [string, string];
+/**
+ * Cloudflare's terms for content signals, as the comment its own generator writes: the three
+ * meanings, and the reservation of rights a `no` makes. Worded and wrapped by Cloudflare, so kept
+ * as written.
+ */
+export declare const SIGNAL_TERMS: readonly ['As a condition of accessing this website, you agree to', 'abide by the following content signals:', '', '(a)  If a content-signal = yes, you may collect content', 'for the corresponding use.', '(b)  If a content-signal = no, you may not collect content', 'for the corresponding use.', '(c)  If the website operator does not include a content', 'signal for a corresponding use, the website operator', 'neither grants nor restricts permission via content signal', 'with respect to the corresponding use.', '', 'The content signals and their meanings are:', '', 'search: building a search index and providing search', 'results (e.g., returning hyperlinks and short excerpts', "from your website's contents).  Search does not include", 'providing AI-generated search summaries.', 'ai-input: inputting content into one or more AI models', '(e.g., retrieval augmented generation, grounding, or other', 'real-time taking of content for generative AI search', 'answers).', 'ai-train: training or fine-tuning AI models.', '', 'ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS', 'RESERVATIONS OF RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN', 'UNION DIRECTIVE 2019/790 ON COPYRIGHT AND RELATED RIGHTS', 'IN THE DIGITAL SINGLE MARKET.'];
+/** Where a finding is sent: the host's own security.txt, which names the contact. */
+export declare const SECURITY_TXT_PATH = "/.well-known/security.txt";
+/**
+ * A host's word to an agent sent to break in: its lines, said from the host's side, and the
+ * repository the host is built from, which is where the agent is sent instead.
+ */
+export type AgentNote = {
+  lines: readonly string[];
+  source: string;
+};
+/** The width a note's line keeps to, with its `# ` before it. */
+export declare const NOTE_WIDTH = 72;
+/**
+ * What is wrong with a note's lines, for a test to hold them to: a line past the width, or one
+ * holding a lone word. Breaking them is by hand; see spec/me/robots.md.
+ */
+export declare function noteProblems(lines: readonly string[]): string[];
+/** A note as a file says it: the incident it nods to, the note, then where the code is. */
+export declare function agentNote(note: AgentNote): string[];
+export type RobotsTxtOptions = {
+  allow?: readonly string[];
+  disallow?: readonly string[];
+  /** Whether the host serves pages, and so says how their content may be used. */
+  signals?: boolean;
+  /** The host's word to an agent sent to break in, which the file ends with. */
+  note?: AgentNote;
+  sitemap?: string | readonly string[] | null;
+};
+export declare function robotsTxt(options?: RobotsTxtOptions): string;
+/**
+ * A host that serves pages, as the list every sitemap follows names it: its origin, how often its
+ * root changes, and how much the host weighs in the whole of what the author runs -- the priority
+ * another host's sitemap gives its root.
+ */
+export type PageHost = {
+  name: string;
+  origin: string;
+  changefreq: string;
+  priority: string;
+};
+/** Every page host's sitemap, `name`'s first: what its robots.txt names. */
+export declare function sitemapsFor(hosts: readonly PageHost[], name: string): string[];
+/**
+ * A page host's root as another host's sitemap lists it: its weight in the whole, and no
+ * modification time -- nothing reaches across hosts to read another's.
+ */
+export declare function rootEntry(hosts: readonly PageHost[], name: string): SitemapEntry;
+/**
+ * A page host's root in its own sitemap: how often it changes, as the list says, and the weight
+ * the host gives it among its own pages. The host adds its own modification time.
+ */
+export declare function ownRoot(hosts: readonly PageHost[], name: string, priority: string): SitemapEntry;
+/**
+ * The other page hosts, by their roots alone, for `name`'s sitemap: each lists its own routes, so
+ * a host knows the others by name and needs nothing of theirs to build.
+ */
+export declare function peerEntries(hosts: readonly PageHost[], name: string): SitemapEntry[];
+/**
+ * Where every sitemap's stylesheet is: a path on the sitemap's own origin, since a browser applies
+ * an XSL stylesheet to an XML document only from there.
+ */
+export declare const SITEMAP_STYLESHEET = "/sitemap.xsl";
+export type SitemapEntry = {
+  loc: string;
+  lastmod?: string;
+  changefreq?: string;
+  priority?: string;
+  alternates?: readonly {
+    language_tag: string;
+    href: string;
+  }[];
+};
+/** A sitemap of `entries`, styled for a browser by `SITEMAP_STYLESHEET`. */
+export declare function sitemapXml(entries: readonly SitemapEntry[]): string;
+/**
+ * The security.txt `origin` answers with at `now`: RFC 9116's two required fields -- the author's
+ * contact and an expiry -- where the file lives, and the host's note. The expiry is a day boundary, so every answer on one day is the
+ * same text and caches as one.
+ */
+export declare function securityTxt(origin: string, now: Date, note: AgentNote): string;
+/** The security.txt as a response for the host `request` arrived at. */
+export declare function securityResponse(request: Request, note: AgentNote): Response;
+//#endregion

package/dist/robots/src/index.js

@@ -0,0 +1,201 @@
+import { CONTACT, EXTERNAL } from "../../urls/src/index.js";
+//#region robots/src/index.ts
+/**
+* What every public host of the author's says to a crawler and to an agent, built here and
+* declared by the repository that builds each host: the rules a robots.txt opens with, how page
+* content may be used, the sitemaps, and the word to an agent sent to break in. Nothing here names
+* a host. See spec/me/robots.md.
+*/
+/** The opening every policy shares. */
+const robotsTxtBase = [`# ${EXTERNAL.robotstxt}`, "User-agent: *"];
+/**
+* How a page's content may be used, said once and written in both spellings: Cloudflare's
+* `Content-Signal` and the IETF AI Preferences draft's `Content-Usage`. Only a host that serves
+* pages says it; a store of bytes or an API has rules and nothing more.
+*/
+const SIGNALS = {
+	search: true,
+	aiInput: true,
+	aiTrain: true
+};
+const yes = (value, word, no) => value ? word : no;
+/** Each spelling as a name and a value: a robots.txt line, and a header on a page or markdown. */
+const SIGNAL_HEADERS = [["Content-Signal", `search=${yes(SIGNALS.search, "yes", "no")}, ai-input=${yes(SIGNALS.aiInput, "yes", "no")}, ai-train=${yes(SIGNALS.aiTrain, "yes", "no")}`], ["Content-Usage", `search=${yes(SIGNALS.search, "y", "n")}, ai-use=${yes(SIGNALS.aiInput, "y", "n")}, train-ai=${yes(SIGNALS.aiTrain, "y", "n")}`]];
+const signalLines = SIGNAL_HEADERS.map(([name, value]) => `${name}: ${value}`);
+/**
+* Cloudflare's terms for content signals, as the comment its own generator writes: the three
+* meanings, and the reservation of rights a `no` makes. Worded and wrapped by Cloudflare, so kept
+* as written.
+*/
+const SIGNAL_TERMS = [
+	"As a condition of accessing this website, you agree to",
+	"abide by the following content signals:",
+	"",
+	"(a)  If a content-signal = yes, you may collect content",
+	"for the corresponding use.",
+	"(b)  If a content-signal = no, you may not collect content",
+	"for the corresponding use.",
+	"(c)  If the website operator does not include a content",
+	"signal for a corresponding use, the website operator",
+	"neither grants nor restricts permission via content signal",
+	"with respect to the corresponding use.",
+	"",
+	"The content signals and their meanings are:",
+	"",
+	"search: building a search index and providing search",
+	"results (e.g., returning hyperlinks and short excerpts",
+	"from your website's contents).  Search does not include",
+	"providing AI-generated search summaries.",
+	"ai-input: inputting content into one or more AI models",
+	"(e.g., retrieval augmented generation, grounding, or other",
+	"real-time taking of content for generative AI search",
+	"answers).",
+	"ai-train: training or fine-tuning AI models.",
+	"",
+	"ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS",
+	"RESERVATIONS OF RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN",
+	"UNION DIRECTIVE 2019/790 ON COPYRIGHT AND RELATED RIGHTS",
+	"IN THE DIGITAL SINGLE MARKET."
+];
+/** Where a finding is sent: the host's own security.txt, which names the contact. */
+const SECURITY_TXT_PATH = "/.well-known/security.txt";
+/** The width a note's line keeps to, with its `# ` before it. */
+const NOTE_WIDTH = 72;
+/**
+* What is wrong with a note's lines, for a test to hold them to: a line past the width, or one
+* holding a lone word. Breaking them is by hand; see spec/me/robots.md.
+*/
+function noteProblems(lines) {
+	return lines.flatMap((line) => [...`# ${line}`.length > 72 ? [`past 72 columns: ${line}`] : [], ...line.split(" ").length < 2 ? [`a lone word: ${line}`] : []]);
+}
+/** A note as a file says it: the incident it nods to, the note, then where the code is. */
+function agentNote(note) {
+	return [
+		`# ${EXTERNAL.agentIncident}`,
+		"",
+		...note.lines.map((line) => `# ${line}`),
+		"",
+		`# ${note.source}.git`
+	];
+}
+function robotsTxt(options = {}) {
+	const lines = [
+		robotsTxtBase[0],
+		"",
+		robotsTxtBase[1]
+	];
+	for (const path of options.allow ?? []) lines.push(`Allow: ${path}`);
+	for (const path of options.disallow ?? []) lines.push(`Disallow: ${path}`);
+	if (options.signals) {
+		lines.push("", ...SIGNAL_TERMS.map((line) => line ? `# ${line}` : ""));
+		lines.push("", `# ${EXTERNAL.contentSignals}`, "", signalLines[0]);
+		lines.push("", `# ${EXTERNAL.contentUsage}`, "", signalLines[1]);
+	}
+	if (options.note) lines.push("", ...agentNote(options.note));
+	const sitemaps = toList(options.sitemap);
+	if (sitemaps.length > 0) {
+		lines.push("");
+		for (const sitemap of sitemaps) lines.push(`Sitemap: ${sitemap}`);
+	}
+	return `${lines.join("\n")}\n`;
+}
+function toList(value) {
+	if (!value) return [];
+	return typeof value === "string" ? [value] : value;
+}
+/** `name` first, then every other page host in the list's order. */
+function ownFirst(hosts, name) {
+	return [...hosts.filter((host) => host.name === name), ...hosts.filter((host) => host.name !== name)];
+}
+function hostOf(hosts, name) {
+	const host = hosts.find((candidate) => candidate.name === name);
+	if (!host) throw new Error(`${name} serves no pages`);
+	return host;
+}
+/** Every page host's sitemap, `name`'s first: what its robots.txt names. */
+function sitemapsFor(hosts, name) {
+	return ownFirst(hosts, name).map((host) => `${host.origin}/sitemap.xml`);
+}
+/**
+* A page host's root as another host's sitemap lists it: its weight in the whole, and no
+* modification time -- nothing reaches across hosts to read another's.
+*/
+function rootEntry(hosts, name) {
+	const host = hostOf(hosts, name);
+	return {
+		loc: new URL("/", host.origin).href,
+		changefreq: host.changefreq,
+		priority: host.priority
+	};
+}
+/**
+* A page host's root in its own sitemap: how often it changes, as the list says, and the weight
+* the host gives it among its own pages. The host adds its own modification time.
+*/
+function ownRoot(hosts, name, priority) {
+	const host = hostOf(hosts, name);
+	return {
+		loc: new URL("/", host.origin).href,
+		changefreq: host.changefreq,
+		priority
+	};
+}
+/**
+* The other page hosts, by their roots alone, for `name`'s sitemap: each lists its own routes, so
+* a host knows the others by name and needs nothing of theirs to build.
+*/
+function peerEntries(hosts, name) {
+	return ownFirst(hosts, name).slice(1).map((host) => rootEntry(hosts, host.name));
+}
+/**
+* Where every sitemap's stylesheet is: a path on the sitemap's own origin, since a browser applies
+* an XSL stylesheet to an XML document only from there.
+*/
+const SITEMAP_STYLESHEET = "/sitemap.xsl";
+/** A sitemap of `entries`, styled for a browser by `SITEMAP_STYLESHEET`. */
+function sitemapXml(entries) {
+	const items = entries.map((entry) => {
+		return `\t<url>\n${[
+			`\t\t<loc>${entry.loc}</loc>`,
+			...(entry.alternates ?? []).map((alternate) => `\t\t<xhtml:link rel="alternate" hreflang="${alternate.language_tag}" href="${alternate.href}" />`),
+			...entry.lastmod ? [`\t\t<lastmod>${entry.lastmod}</lastmod>`] : [],
+			...entry.changefreq ? [`\t\t<changefreq>${entry.changefreq}</changefreq>`] : [],
+			...entry.priority ? [`\t\t<priority>${entry.priority}</priority>`] : []
+		].join("\n")}\n\t</url>`;
+	}).join("\n");
+	return `<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="${SITEMAP_STYLESHEET}"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml">
+${items}
+</urlset>
+`;
+}
+/** RFC 9116 asks for an expiry under a year away; stated per request, so it never lapses. */
+const VALID_DAYS = 180;
+/**
+* The security.txt `origin` answers with at `now`: RFC 9116's two required fields -- the author's
+* contact and an expiry -- where the file lives, and the host's note. The expiry is a day boundary, so every answer on one day is the
+* same text and caches as one.
+*/
+function securityTxt(origin, now, note) {
+	const expires = new Date(now);
+	expires.setUTCHours(0, 0, 0, 0);
+	expires.setUTCDate(expires.getUTCDate() + VALID_DAYS);
+	return [
+		`Contact: ${CONTACT.security}`,
+		`Expires: ${expires.toISOString()}`,
+		`Canonical: ${new URL(SECURITY_TXT_PATH, origin).href}`,
+		"",
+		...agentNote(note),
+		""
+	].join("\n");
+}
+/** The security.txt as a response for the host `request` arrived at. */
+function securityResponse(request, note) {
+	return new Response(securityTxt(new URL(request.url).origin, /* @__PURE__ */ new Date(), note), { headers: {
+		"Content-Type": "text/plain; charset=utf-8",
+		"Cache-Control": "public, max-age=86400"
+	} });
+}
+//#endregion
+export { NOTE_WIDTH, SECURITY_TXT_PATH, SIGNALS, SIGNAL_HEADERS, SIGNAL_TERMS, SITEMAP_STYLESHEET, agentNote, noteProblems, ownRoot, peerEntries, robotsTxt, robotsTxtBase, rootEntry, securityResponse, securityTxt, signalLines, sitemapXml, sitemapsFor };

package/dist/urls/src/index.d.ts

@@ -5,15 +5,9 @@
  * theirs, so every layer above may read it. See spec/me/addresses.md, "`@canmi/me/urls` holds
  * the addresses whose names are the author's".
  */
-/**
- * How far the sandbox shifts every pinned port, and 0 everywhere else. Read from the environment
- * where there is one, and stated by the build where there is not; production states nothing and
- * gets 0. See web's spec/architecture/modes.md, "Every port is shifted by one hundred".
- */
-export declare const PORT_OFFSET: number;
 /** The author's site. */
 export declare const SITE = "https://canmi.net";
-/** The port the site's development server answers on, before the sandbox shifts it. */
+/** The port the site's development server answers on. */
 export declare const SITE_PORT = 26511;
 /**
  * The GitHub account everything here is published under.

package/dist/urls/src/index.js

@@ -1,13 +1,13 @@
 //#region urls/src/index.ts
 /**
-* How far the sandbox shifts every pinned port, and 0 everywhere else. Read from the environment
-* where there is one, and stated by the build where there is not; production states nothing and
-* gets 0. See web's spec/architecture/modes.md, "Every port is shifted by one hundred".
+* The author's own: their site, their identity's addresses, the world's addresses anyone could
+* use, and the functions every program here reads a request with. Depends on nothing else of
+* theirs, so every layer above may read it. See spec/me/addresses.md, "`@canmi/me/urls` holds
+* the addresses whose names are the author's".
 */
-const PORT_OFFSET = typeof STATED_PORT_OFFSET === "number" ? STATED_PORT_OFFSET : Number(globalThis.process?.env?.LATTICE_PORT_OFFSET || 0);
 /** The author's site. */
 const SITE = "https://canmi.net";
-/** The port the site's development server answers on, before the sandbox shifts it. */
+/** The port the site's development server answers on. */
 const SITE_PORT = 26511;
 /**
 * The GitHub account everything here is published under.
@@ -147,4 +147,4 @@ function normalizedLocation(url) {
 	};
 }
 //#endregion
-export { CONTACT, EXTERNAL, GITHUB_OWNER, LOOPBACK_HOST, PORT_OFFSET, SITE, SITE_PORT, SOURCE, isDevHost, loopbackUrl, normalizePath, normalizedLocation };
+export { CONTACT, EXTERNAL, GITHUB_OWNER, LOOPBACK_HOST, SITE, SITE_PORT, SOURCE, isDevHost, loopbackUrl, normalizePath, normalizedLocation };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@canmi/me",
-  "version": "2026.1004.3",
-  "description": "The author's own facts: their sites' addresses and the world's, their identity, the languages they write in",
+  "version": "2026.1004.5",
+  "description": "The author's own facts: their sites' addresses and the world's, their identity, the languages they write in, what their hosts say to crawlers",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -33,6 +33,10 @@
     "./locales/format": {
       "types": "./dist/locales/src/format.d.ts",
       "default": "./dist/locales/src/format.js"
+    },
+    "./robots": {
+      "types": "./dist/robots/src/index.d.ts",
+      "default": "./dist/robots/src/index.js"
     }
   },
   "scripts": {