llms-ink 0.2.0 → 0.3.0

package/README.md

@@ -52,7 +52,27 @@ Building a big site or a video can take minutes. Bound a call with a signal:
 ### `ink.check(url, { signal? })` → `Promise<CheckResult>`
 
 Checks the site's (or docs section's) published llms.txt against the llmstxt.org format and fetches its first
-25 links: `{ site, url, found, status, title, tokens, sections, links, llmsFull, issues, summary, checkedAt }`.
+25 links: `{ site, url, found, status, title, tokens, sections, links, llmsFull, parsed, issues, summary, checkedAt }`.
+
+`parsed` is the file's structure (`null` when nothing was found):
+
+```ts
+{ title, summary, details, sections: [{ name, optional, text, links: [{ title, url, notes, line }] }] }
+```
+
+### `ink.checkText(text, { base?, links?, signal? })` → `Promise<CheckResult>`
+
+Parses and checks llms.txt text you already have, e.g. before publishing it:
+
+```ts
+const { parsed, issues } = await ink.checkText(await readFile('public/llms.txt', 'utf8'), {
+  base: 'https://example.com/llms.txt', // resolves relative links; reported as url/site
+  links: true,                          // also fetch up to 25 links and look for llms-full.txt
+});
+```
+
+Without `base`, `site` and `url` are `null`; without `base` and `links`, `llmsFull` is `null` and
+`links.checked` is 0. Up to 2 MB.
 
 ### `ink.stream(url, { mode?, fresh?, signal? })` → `AsyncGenerator<StreamEvent>`
 

package/dist/index.d.ts

@@ -13,7 +13,7 @@
  *
  * @module
  */
-export declare const VERSION = "0.2.0";
+export declare const VERSION = "0.3.0";
 export declare const DEFAULT_BASE_URL = "https://llms.ink";
 /** `llms` for the index (llms.txt), `full` for llms-full.txt. */
 export type Mode = 'llms' | 'full';
@@ -72,12 +72,40 @@ export interface CheckIssue {
     line: number | null;
     message: string;
 }
-/** `GET /api/v0/check`: a published llms.txt checked against the format and its links. */
-export interface CheckResult {
-    /** The site's origin. */
-    site: string;
-    /** The llms.txt that was checked (or the last place looked when none was found). */
+/** One link of a section: `- [title](url): notes`. */
+export interface ParsedLink {
+    title: string;
+    /** Resolved against `base` / the fetched URL when possible, else as written. */
     url: string;
+    /** Text after `:`, trimmed. */
+    notes: string | null;
+    line: number;
+}
+/** An H2 section of llms.txt. */
+export interface ParsedSection {
+    name: string;
+    /** The section is named "Optional" (case-insensitive). */
+    optional: boolean;
+    /** Non-link content of the section. */
+    text: string | null;
+    links: ParsedLink[];
+}
+/** The structure of an llms.txt file. */
+export interface ParsedLlms {
+    /** The H1. */
+    title: string | null;
+    /** The blockquote, without `> `. */
+    summary: string | null;
+    /** Markdown between the summary and the first H2. */
+    details: string | null;
+    sections: ParsedSection[];
+}
+/** `GET` / `POST /api/v0/check`: an llms.txt checked against the format and its links. */
+export interface CheckResult {
+    /** The site's origin; null for {@link LlmsInk.checkText} without `base`. */
+    site: string | null;
+    /** The llms.txt that was checked (or the last place looked when none was found); null for text without `base`. */
+    url: string | null;
     found: boolean;
     /** HTTP status of the llms.txt request. */
     status: number;
@@ -87,7 +115,7 @@ export interface CheckResult {
     sections: number;
     links: {
         total: number;
-        /** Links fetched this run (the first 25). */
+        /** Links fetched this run (the first 25); 0 when links weren't checked. */
         checked: number;
         /** `status` is 0 when the link didn't answer in time. */
         broken: Array<{
@@ -95,8 +123,10 @@ export interface CheckResult {
             status: number;
         }>;
     };
-    /** Whether llms-full.txt is published next to it. */
-    llmsFull: boolean;
+    /** Whether llms-full.txt is published next to it; null when not checked. */
+    llmsFull: boolean | null;
+    /** The parsed file; null when nothing was found. */
+    parsed: ParsedLlms | null;
     /** Errors first, then warnings and notes, in line order. */
     issues: CheckIssue[];
     summary: {
@@ -206,6 +236,14 @@ export interface LlmsOptions {
 export interface CheckOptions {
     signal?: AbortSignal;
 }
+/** Options for {@link LlmsInk.checkText}. */
+export interface CheckTextOptions {
+    /** Public URL the text lives at: resolves relative links and is reported as `url` / `site`. */
+    base?: string;
+    /** Also check up to 25 linked URLs (and llms-full.txt next to `base`). Defaults to `false`. */
+    links?: boolean;
+    signal?: AbortSignal;
+}
 /**
  * A client for https://llms.ink.
  *
@@ -226,6 +264,8 @@ export declare class LlmsInk {
     llms(url: string, options?: LlmsOptions): Promise<Result>;
     /** Checks a site's published llms.txt against the llmstxt.org format and that its links answer. */
     check(url: string, options?: CheckOptions): Promise<CheckResult>;
+    /** Parses and checks llms.txt text you already have (`POST /api/v0/check`); nothing is published. */
+    checkText(text: string, options?: CheckTextOptions): Promise<CheckResult>;
     /**
      * Like {@link LlmsInk.llms}, as the build happens: yields `progress` and `partial` events, then one
      * `result`. Errors — including a terminal `error` event — are thrown as {@link LlmsInkError}s.

package/dist/index.js

@@ -14,7 +14,7 @@
  * @module
  */
 import { env } from "./env.js";
-export const VERSION = '0.2.0';
+export const VERSION = '0.3.0';
 export const DEFAULT_BASE_URL = 'https://llms.ink';
 /**
  * An API error, or llms.ink couldn't be reached (then `status` and `code` are undefined).
@@ -99,12 +99,22 @@ export class LlmsInk {
      * any public link.
      */
     async llms(url, options = {}) {
-        const res = await this.#get('/api/v0/llms', llmsParams(url, options), 'application/json', options.signal);
+        const res = await this.#request('/api/v0/llms', llmsParams(url, options), 'application/json', options.signal);
         return (await this.#json(res));
     }
     /** Checks a site's published llms.txt against the llmstxt.org format and that its links answer. */
     async check(url, options = {}) {
-        const res = await this.#get('/api/v0/check', { url }, 'application/json', options.signal);
+        const res = await this.#request('/api/v0/check', { url }, 'application/json', options.signal);
+        return (await this.#json(res));
+    }
+    /** Parses and checks llms.txt text you already have (`POST /api/v0/check`); nothing is published. */
+    async checkText(text, options = {}) {
+        const params = {};
+        if (options.base)
+            params.base = options.base;
+        if (options.links)
+            params.links = '1';
+        const res = await this.#request('/api/v0/check', params, 'application/json', options.signal, text);
         return (await this.#json(res));
     }
     /**
@@ -113,7 +123,7 @@ export class LlmsInk {
      * Breaking out of the loop cancels the request.
      */
     async *stream(url, options = {}) {
-        const res = await this.#get('/api/v0/llms', llmsParams(url, options), 'application/x-ndjson', options.signal);
+        const res = await this.#request('/api/v0/llms', llmsParams(url, options), 'application/x-ndjson', options.signal);
         if (!res.headers.get('content-type')?.includes('ndjson')) {
             yield { type: 'result', result: (await this.#json(res)) };
             return;
@@ -173,16 +183,19 @@ export class LlmsInk {
         const known = event.type === 'progress' || event.type === 'partial' || event.type === 'result';
         return known ? event : undefined;
     }
-    async #get(path, params, accept, signal) {
+    async #request(path, params, accept, signal, body) {
         const headers = { accept };
+        if (body !== undefined)
+            headers['content-type'] = 'text/markdown; charset=utf-8';
         if (!IS_BROWSER)
             headers['user-agent'] = `llms-ink-js/${VERSION}`;
         if (this.key)
             headers.authorization = `Bearer ${this.key}`;
-        const target = `${this.baseUrl}${path}?${new URLSearchParams(params)}`;
+        const query = new URLSearchParams(params).toString();
+        const target = `${this.baseUrl}${path}${query ? `?${query}` : ''}`;
         let res;
         try {
-            res = await this.#fetch(target, { headers, signal });
+            res = await this.#fetch(target, body === undefined ? { headers, signal } : { method: 'POST', headers, body, signal });
         }
         catch (err) {
             if (isAbort(err))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llms-ink",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Turn any link — site, docs section, page, GitHub repo, PDF or video — into llms.txt and llms-full.txt. Typed client for llms.ink, plus a Vite plugin, an Astro integration and middleware that serve the files from your site. No dependencies.",
   "type": "module",
   "exports": {
@@ -25,6 +25,8 @@
   "types": "./dist/index.d.ts",
   "files": ["dist"],
   "sideEffects": false,
+  "repository": { "type": "git", "url": "git+https://github.com/lukax/llms-ink-js.git" },
+  "bugs": "https://github.com/lukax/llms-ink-js/issues",
   "engines": { "node": ">=18" },
   "scripts": {
     "build": "rm -rf dist && tsc -p tsconfig.json",