@nshiab/setup-data-project 2.0.0 → 2.0.1

package/README.md

@@ -25,7 +25,8 @@ published. It is tracked for removal in
   the full
   [simple-data-analysis](https://github.com/nshiab/simple-data-analysis/)
   package, and [journalism](https://github.com/nshiab/journalism) libraries,
-  then fetches their documentation for LLM use.
+  then fetches their README files and complete API documentation from the
+  matching version tags on GitHub for LLM use.
 - Updates project configuration (e.g., `deno.json` or `package.json`) with
   relevant tasks.
 

package/esm/helpers/ensureAgents.d.ts

@@ -1,2 +1,3 @@
-export declare function ensureAgents(docsMapping: Record<string, string>, runtime: string, installedPackages?: string[]): void;
+import type { PackageDocs } from "./fetchPackageDocs.js";
+export declare function ensureAgents(docsMapping: Record<string, PackageDocs>, runtime: string, installedPackages?: string[]): void;
 //# sourceMappingURL=ensureAgents.d.ts.map

package/esm/helpers/ensureAgents.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ensureAgents.d.ts","sourceRoot":"","sources":["../../src/helpers/ensureAgents.ts"],"names":[],"mappings":"AAOA,wBAAgB,YAAY,CAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACnC,OAAO,EAAE,MAAM,EACf,iBAAiB,GAAE,MAAM,EAA6B,QAmJvD"}
+{"version":3,"file":"ensureAgents.d.ts","sourceRoot":"","sources":["../../src/helpers/ensureAgents.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAOzD,wBAAgB,YAAY,CAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACxC,OAAO,EAAE,MAAM,EACf,iBAAiB,GAAE,MAAM,EAA6B,QAqLvD"}

package/esm/helpers/ensureAgents.js

@@ -5,11 +5,15 @@ export function ensureAgents(docsMapping, runtime, installedPackages = Object.ke
     let journalismFunctions = "";
     let sdaClassesAndMethods = "";
     let sdaDocsPath = "";
+    let sdaReadmePath = "";
     let sdaLibraryName = "";
     const journalismFunctionsByPackage = new Map();
-    for (const [pkg, doc] of Object.entries(docsMapping)) {
+    for (const [pkg, docs] of Object.entries(docsMapping)) {
         const pkgConfig = SUPPORTED_PACKAGES.find((p) => p.value === pkg);
         const pkgType = pkgConfig?.type || "other";
+        const doc = docs.llm;
+        if (doc === undefined)
+            continue;
         if (pkgType === "journalism") {
             const repoName = pkg.split("/")[1];
             const functions = doc
@@ -27,7 +31,10 @@ export function ensureAgents(docsMapping, runtime, installedPackages = Object.ke
         }
         else if (pkgType === "sda") {
             const repoName = pkg.split("/")[1];
-            sdaDocsPath = `./docs/${repoName}.md`;
+            sdaDocsPath = `./docs/${repoName}/llm.md`;
+            sdaReadmePath = docs.readme === undefined
+                ? ""
+                : `./docs/${repoName}/README.md`;
             sdaLibraryName = pkgConfig?.label || repoName;
             sdaClassesAndMethods = doc
                 .split("\n")
@@ -89,6 +96,30 @@ export function ensureAgents(docsMapping, runtime, installedPackages = Object.ke
         libraryGuidance = `
 Always prioritize the installed ${libraryNames} ${libraryNoun} when relevant.
 `;
+        const documentationGuidance = installedPackageConfigs.flatMap((pkg) => {
+            const docs = docsMapping[pkg.value];
+            if (!docs)
+                return [];
+            const repoName = pkg.value.split("/")[1];
+            const availableDocs = [
+                docs.readme === undefined
+                    ? ""
+                    : `overview and practical examples at \`./docs/${repoName}/README.md\``,
+                docs.llm === undefined
+                    ? ""
+                    : `complete API documentation at \`./docs/${repoName}/llm.md\``,
+            ].filter(Boolean);
+            return availableDocs.length === 0
+                ? []
+                : [`- \`${pkg.value}\`: ${availableDocs.join("; ")}`];
+        });
+        if (documentationGuidance.length > 0) {
+            libraryGuidance += `
+The following documentation was fetched for the exact installed library versions. Consult it when relevant:
+
+${documentationGuidance.join("\n")}
+`;
+        }
         if (importExamples.length > 0) {
             libraryGuidance += `APIs can be imported with named imports like this:
 \`\`\`typescript
@@ -117,12 +148,15 @@ If you need to output data to a file, always put the file in the "sda/output" fo
 ${libraryGuidance}`;
     if (journalismFunctions !== "") {
         content += `
-Here are the functions available in the "journalism" libraries. If one of the function might be relevant, read the complete documentation in the "./docs/" folder to properly use it.
+Here are the functions available in the "journalism" libraries. If one of the functions might be relevant, search its complete API documentation in the corresponding \`./docs/<library-name>/llm.md\` file to properly use it.
 ${journalismFunctions}`;
     }
     if (sdaClassesAndMethods !== "") {
+        const readmeGuidance = sdaReadmePath === ""
+            ? ""
+            : ` An overview and practical examples are available at "${sdaReadmePath}".`;
         content += `
-Here are the classes and their methods available in the "${sdaLibraryName}" library. If one of the classes or methods might be relevant, read the complete documentation at "${sdaDocsPath}" to properly use it.
+Here are the classes and their methods available in the "${sdaLibraryName}" library.${readmeGuidance} If one of the classes or methods might be relevant, search the complete API documentation at "${sdaDocsPath}" to properly use it.
 
 Most data-loading and transformation methods are synchronous, chainable builders. Await the final asynchronous observer or export method, such as \`log()\`, \`getData()\`, or \`writeData()\`, to execute the queued operations. If a chain ends without an observer, call \`run()\`. Always call \`await sdb.close()\` when the database is no longer needed. Methods that return an answer or export to an external service can be asynchronous, so verify their documentation before calling them.
 ${sdaClassesAndMethods}`;

package/esm/helpers/fetchPackageDocs.d.ts

@@ -0,0 +1,8 @@
+export interface PackageDocs {
+    readme?: string;
+    llm?: string;
+}
+export declare function fetchPackageDocs(pkg: string, version: string, options?: {
+    silent?: boolean;
+}): Promise<PackageDocs>;
+//# sourceMappingURL=fetchPackageDocs.d.ts.map

package/esm/helpers/fetchPackageDocs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetchPackageDocs.d.ts","sourceRoot":"","sources":["../../src/helpers/fetchPackageDocs.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAiBD,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAO,GACjC,OAAO,CAAC,WAAW,CAAC,CA4DtB"}

package/esm/helpers/fetchPackageDocs.js

@@ -0,0 +1,75 @@
+import { log, spinner } from "@clack/prompts";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+const PACKAGE_DOCUMENTS = [
+    {
+        remoteName: "README.md",
+        localName: "README.md",
+        key: "readme",
+        label: "README",
+    },
+    {
+        remoteName: "llm.md",
+        localName: "llm.md",
+        key: "llm",
+        label: "API documentation",
+    },
+];
+export async function fetchPackageDocs(pkg, version, options = {}) {
+    const repoName = pkg.split("/")[1];
+    const docsDirectory = join("docs", repoName);
+    mkdirSync(docsDirectory, { recursive: true });
+    const sFetch = spinner();
+    if (!options.silent) {
+        sFetch.start(`Fetching documentation for ${pkg}@${version}...`);
+    }
+    const fetchedDocuments = await Promise.all(PACKAGE_DOCUMENTS.map(async (document) => {
+        const localPath = join(docsDirectory, document.localName);
+        try {
+            const url = "https://raw.githubusercontent.com/nshiab/" + repoName +
+                `/refs/tags/v${version}/${document.remoteName}`;
+            const response = await fetch(url);
+            if (!response.ok) {
+                warnAboutFailedFetch(pkg, version, document.label, localPath);
+                return { key: document.key, content: undefined };
+            }
+            const content = await response.text();
+            writeFileSync(localPath, content);
+            return { key: document.key, content };
+        }
+        catch (error) {
+            warnAboutFailedFetch(pkg, version, document.label, localPath);
+            console.error(error);
+            return { key: document.key, content: undefined };
+        }
+    }));
+    const docs = {};
+    for (const document of fetchedDocuments) {
+        if (document.content !== undefined) {
+            docs[document.key] = document.content;
+        }
+    }
+    const savedCount = fetchedDocuments.filter((document) => document.content !== undefined)
+        .length;
+    if (!options.silent) {
+        if (savedCount === PACKAGE_DOCUMENTS.length) {
+            sFetch.stop(`✅ Documentation for ${pkg}@${version} saved!`);
+        }
+        else if (savedCount > 0) {
+            sFetch.stop(`⚠️  Some documentation for ${pkg}@${version} could not be fetched.`);
+        }
+        else {
+            sFetch.stop(`❌ Failed to fetch documentation for ${pkg}@${version}.`);
+        }
+    }
+    return docs;
+}
+function warnAboutFailedFetch(pkg, version, label, localPath) {
+    const existingFileGuidance = existsSync(localPath)
+        ? ` The existing file at ${localPath} was preserved, but it may not ` +
+            "match the installed version. Double-check it before relying on it, or " +
+            "remove it."
+        : "";
+    log.warn(`Could not fetch ${label} for ${pkg}@${version} from its GitHub tag.` +
+        existingFileGuidance);
+}

package/esm/helpers/getInstalledPackageVersions.d.ts

@@ -0,0 +1,2 @@
+export declare function getInstalledPackageVersions(packageSpecifiers: Record<string, string>): Record<string, string>;
+//# sourceMappingURL=getInstalledPackageVersions.d.ts.map

package/esm/helpers/getInstalledPackageVersions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"getInstalledPackageVersions.d.ts","sourceRoot":"","sources":["../../src/helpers/getInstalledPackageVersions.ts"],"names":[],"mappings":"AAKA,wBAAgB,2BAA2B,CACzC,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACxC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAuBxB"}

package/esm/helpers/getInstalledPackageVersions.js

@@ -0,0 +1,91 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+export function getInstalledPackageVersions(packageSpecifiers) {
+    const denoLock = readJsonFile("deno.lock");
+    const packageLock = readJsonFile("package-lock.json");
+    const bunLock = existsSync("bun.lock")
+        ? readFileSync("bun.lock", "utf-8")
+        : undefined;
+    const versions = {};
+    for (const [pkg, specifier] of Object.entries(packageSpecifiers)) {
+        const version = specifier.startsWith("jsr:")
+            ? getDenoLockVersion(pkg, specifier, denoLock) ??
+                getExactVersion(specifier)
+            : getNodeModulesVersion(pkg) ??
+                getPackageLockVersion(pkg, packageLock) ??
+                getBunLockVersion(pkg, bunLock) ??
+                getExactVersion(specifier);
+        if (version !== undefined) {
+            versions[pkg] = version;
+        }
+    }
+    return versions;
+}
+function getNodeModulesVersion(pkg) {
+    const packageJson = readJsonFile(join("node_modules", ...pkg.split("/"), "package.json"));
+    return typeof packageJson?.version === "string"
+        ? getExactVersion(packageJson.version)
+        : undefined;
+}
+function getDenoLockVersion(pkg, specifier, lock) {
+    if (!lock || !specifier.startsWith("jsr:"))
+        return undefined;
+    const specifiers = asObject(lock.specifiers);
+    if (!specifiers)
+        return undefined;
+    const directVersion = specifiers[specifier];
+    if (typeof directVersion === "string")
+        return getExactVersion(directVersion);
+    const prefix = `jsr:${pkg}@`;
+    const matchingVersions = Object.entries(specifiers)
+        .filter(([key, value]) => key.startsWith(prefix) && typeof value === "string")
+        .flatMap(([, value]) => {
+        const version = getExactVersion(value);
+        return version === undefined ? [] : [version];
+    });
+    const uniqueVersions = [...new Set(matchingVersions)];
+    return uniqueVersions.length === 1 ? uniqueVersions[0] : undefined;
+}
+function getPackageLockVersion(pkg, lock) {
+    if (!lock)
+        return undefined;
+    const packages = asObject(lock.packages);
+    const packageEntry = packages
+        ? asObject(packages[`node_modules/${pkg}`])
+        : undefined;
+    if (typeof packageEntry?.version === "string") {
+        return getExactVersion(packageEntry.version);
+    }
+    const dependencies = asObject(lock.dependencies);
+    const dependencyEntry = dependencies
+        ? asObject(dependencies[pkg])
+        : undefined;
+    return typeof dependencyEntry?.version === "string"
+        ? getExactVersion(dependencyEntry.version)
+        : undefined;
+}
+function getBunLockVersion(pkg, lock) {
+    if (!lock)
+        return undefined;
+    const escapedPackage = pkg.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const match = lock.match(new RegExp(`"${escapedPackage}"\\s*:\\s*\\[\\s*"${escapedPackage}@(\\d+\\.\\d+\\.\\d+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)"`));
+    return match?.[1];
+}
+function getExactVersion(specifier) {
+    return specifier.match(/(?:^|@)(\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)$/)?.[1];
+}
+function readJsonFile(path) {
+    if (!existsSync(path))
+        return undefined;
+    try {
+        return asObject(JSON.parse(readFileSync(path, "utf-8")));
+    }
+    catch {
+        return undefined;
+    }
+}
+function asObject(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : undefined;
+}

package/esm/helpers/installPackages.d.ts

@@ -0,0 +1,7 @@
+export declare const commandRunner: {
+    exec: any;
+};
+export declare function installPackages(selectedPackages: string[], options?: {
+    silent?: boolean;
+}): Promise<void>;
+//# sourceMappingURL=installPackages.d.ts.map

package/esm/helpers/installPackages.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"installPackages.d.ts","sourceRoot":"","sources":["../../src/helpers/installPackages.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,aAAa;;CAEzB,CAAC;AASF,wBAAsB,eAAe,CACnC,gBAAgB,EAAE,MAAM,EAAE,EAC1B,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAO,GACjC,OAAO,CAAC,IAAI,CAAC,CAmDf"}

package/esm/helpers/installPackages.js

@@ -0,0 +1,59 @@
+import { spinner } from "@clack/prompts";
+import * as childProcess from "node:child_process";
+import { getRuntime } from "./getRuntime.js";
+export const commandRunner = {
+    exec: childProcess.exec,
+};
+// Temporary: remove this bypass after the 2.0 release window.
+// https://github.com/nshiab/setup-data-project/issues/15
+const PACKAGES_WITH_FRESH_DENO_RELEASES = new Set([
+    "@nshiab/simple-data-analysis-core",
+    "@nshiab/simple-data-analysis",
+]);
+export async function installPackages(selectedPackages, options = {}) {
+    const runtime = getRuntime();
+    for (const selectedPackage of selectedPackages) {
+        const packagesToInstall = [selectedPackage];
+        if (selectedPackage === "@nshiab/simple-data-analysis" ||
+            selectedPackage === "@nshiab/journalism-dataviz") {
+            packagesToInstall.push("@observablehq/plot");
+        }
+        for (const pkg of packagesToInstall) {
+            const s = spinner();
+            const isObservablePlot = pkg === "@observablehq/plot";
+            const minimumDependencyAgeOption = PACKAGES_WITH_FRESH_DENO_RELEASES.has(pkg) ? " --min-dep-age=0" : "";
+            const installCmd = runtime === "deno"
+                ? (isObservablePlot
+                    ? "deno add npm:" + pkg
+                    : "deno add" + minimumDependencyAgeOption + " jsr:" +
+                        pkg)
+                : runtime === "bun"
+                    ? "bun add " + pkg
+                    : "npm install " + pkg;
+            if (!options.silent) {
+                s.start("Installing " + pkg + " via " + runtime + "...");
+            }
+            try {
+                await new Promise((resolve, reject) => {
+                    commandRunner.exec(installCmd, (error) => {
+                        if (error) {
+                            reject(error);
+                        }
+                        else {
+                            resolve();
+                        }
+                    });
+                });
+                if (!options.silent) {
+                    s.stop("✅ " + pkg + " installed!");
+                }
+            }
+            catch (error) {
+                if (!options.silent) {
+                    s.stop("❌ Failed to install " + pkg + ".");
+                }
+                throw error;
+            }
+        }
+    }
+}

package/esm/helpers/projectManifest.d.ts

@@ -5,6 +5,7 @@ export declare class ProjectManifest {
     static open(runtime: Runtime): ProjectManifest;
     reload(): void;
     getInstalledPackages(): string[];
+    getInstalledPackageSpecifiers(): Record<string, string>;
     updateTasks(tasks: Record<string, string>): void;
 }
 export {};

package/esm/helpers/projectManifest.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"projectManifest.d.ts","sourceRoot":"","sources":["../../src/helpers/projectManifest.ts"],"names":[],"mappings":"AAEA,KAAK,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;AAGvC,qBAAa,eAAe;;IAM1B,OAAO;IAYP,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,GAAG,eAAe;IA6B9C,MAAM,IAAI,IAAI;IAId,oBAAoB,IAAI,MAAM,EAAE;IAgBhC,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;CAcjD"}
+{"version":3,"file":"projectManifest.d.ts","sourceRoot":"","sources":["../../src/helpers/projectManifest.ts"],"names":[],"mappings":"AAEA,KAAK,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;AAGvC,qBAAa,eAAe;;IAM1B,OAAO;IAYP,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,GAAG,eAAe;IA6B9C,MAAM,IAAI,IAAI;IAId,oBAAoB,IAAI,MAAM,EAAE;IAIhC,6BAA6B,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAoBvD,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;CAcjD"}

package/esm/helpers/projectManifest.js

@@ -37,12 +37,19 @@ export class ProjectManifest {
         __classPrivateFieldSet(this, _ProjectManifest_config, readManifest(__classPrivateFieldGet(this, _ProjectManifest_path, "f")), "f");
     }
     getInstalledPackages() {
-        const installed = [];
+        return Object.keys(this.getInstalledPackageSpecifiers());
+    }
+    getInstalledPackageSpecifiers() {
+        const installed = {};
         for (const key of __classPrivateFieldGet(this, _ProjectManifest_dependencyKeys, "f")) {
             const dependencies = __classPrivateFieldGet(this, _ProjectManifest_config, "f")[key];
             if (dependencies && typeof dependencies === "object" &&
                 !Array.isArray(dependencies)) {
-                installed.push(...Object.keys(dependencies));
+                for (const [name, specifier] of Object.entries(dependencies)) {
+                    if (typeof specifier === "string") {
+                        installed[name] = specifier;
+                    }
+                }
             }
         }
         return installed;

package/esm/helpers/syncPackageDocs.d.ts

@@ -1,2 +1,3 @@
-export declare function syncPackageDocs(installedPackages: string[], docsAlreadyRefreshed?: string[]): Promise<Record<string, string>>;
+import { type PackageDocs } from "./fetchPackageDocs.js";
+export declare function syncPackageDocs(installedPackages: string[], installedVersions: Record<string, string>): Promise<Record<string, PackageDocs>>;
 //# sourceMappingURL=syncPackageDocs.d.ts.map

package/esm/helpers/syncPackageDocs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"syncPackageDocs.d.ts","sourceRoot":"","sources":["../../src/helpers/syncPackageDocs.ts"],"names":[],"mappings":"AAMA,wBAAsB,eAAe,CACnC,iBAAiB,EAAE,MAAM,EAAE,EAC3B,oBAAoB,GAAE,MAAM,EAAO,GAClC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CA0BjC"}
+{"version":3,"file":"syncPackageDocs.d.ts","sourceRoot":"","sources":["../../src/helpers/syncPackageDocs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAoB,KAAK,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAG3E,wBAAsB,eAAe,CACnC,iBAAiB,EAAE,MAAM,EAAE,EAC3B,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACxC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CA+CtC"}

package/esm/helpers/syncPackageDocs.js

@@ -1,26 +1,45 @@
 import { log } from "@clack/prompts";
-import { existsSync, readFileSync, rmSync } from "node:fs";
+import { existsSync, mkdirSync, renameSync, rmSync } from "node:fs";
 import { join } from "node:path";
-import { fetchPackageDocs } from "./installPackagesAndFetchDocs.js";
+import { fetchPackageDocs } from "./fetchPackageDocs.js";
 import { SUPPORTED_PACKAGES } from "./packageRegistry.js";
-export async function syncPackageDocs(installedPackages, docsAlreadyRefreshed = []) {
+export async function syncPackageDocs(installedPackages, installedVersions) {
     const installedSet = new Set(installedPackages);
-    const refreshedSet = new Set(docsAlreadyRefreshed);
     const docsMapping = {};
     for (const pkg of SUPPORTED_PACKAGES) {
-        const docPath = join("docs", pkg.value.split("/")[1] + ".md");
+        const repoName = pkg.value.split("/")[1];
+        const docsDirectory = join("docs", repoName);
+        const llmDocsPath = join(docsDirectory, "llm.md");
+        const legacyDocsPath = join("docs", repoName + ".md");
         if (!installedSet.has(pkg.value)) {
-            if (existsSync(docPath)) {
-                rmSync(docPath);
-                log.info(`Removed stale documentation at ${docPath}.`);
+            for (const path of [docsDirectory, legacyDocsPath]) {
+                if (existsSync(path)) {
+                    rmSync(path, { recursive: true });
+                    log.info(`Removed stale documentation at ${path}.`);
+                }
             }
             continue;
         }
-        if (!existsSync(docPath) && !refreshedSet.has(pkg.value)) {
-            await fetchPackageDocs(pkg.value, { silent: true });
+        if (existsSync(legacyDocsPath)) {
+            if (!existsSync(llmDocsPath)) {
+                mkdirSync(docsDirectory, { recursive: true });
+                renameSync(legacyDocsPath, llmDocsPath);
+                log.info(`Moved documentation to ${llmDocsPath}.`);
+            }
+            else {
+                rmSync(legacyDocsPath);
+            }
+        }
+        const version = installedVersions[pkg.value];
+        if (version === undefined) {
+            log.warn(`Could not determine the exact installed version of ${pkg.value}. ` +
+                "Existing local documentation was preserved but will not be " +
+                "included in AGENTS.md.");
+            continue;
         }
-        if (existsSync(docPath)) {
-            docsMapping[pkg.value] = readFileSync(docPath, "utf-8");
+        const docs = await fetchPackageDocs(pkg.value, version, { silent: true });
+        if (docs.readme !== undefined || docs.llm !== undefined) {
+            docsMapping[pkg.value] = docs;
         }
     }
     return docsMapping;

package/esm/main.js

@@ -5,7 +5,7 @@ import { getRuntime } from "./helpers/getRuntime.js";
 import { PACKAGE_OPTIONS } from "./helpers/packageOptions.js";
 import { ensureGitignore } from "./helpers/ensureGitignore.js";
 import { ensureReadme } from "./helpers/ensureReadme.js";
-import { installPackagesAndFetchDocs } from "./helpers/installPackagesAndFetchDocs.js";
+import { installPackages } from "./helpers/installPackages.js";
 import { getProjectTasks } from "./helpers/getProjectTasks.js";
 import { createFolderStructure } from "./helpers/createFolderStructure.js";
 import { ensureEnvFile } from "./helpers/ensureEnvFile.js";
@@ -13,6 +13,7 @@ import { ensureAgents } from "./helpers/ensureAgents.js";
 import { getInstalledPackageConfigs } from "./helpers/packageRegistry.js";
 import { ProjectManifest } from "./helpers/projectManifest.js";
 import { syncPackageDocs } from "./helpers/syncPackageDocs.js";
+import { getInstalledPackageVersions } from "./helpers/getInstalledPackageVersions.js";
 async function main() {
     const runtime = getRuntime();
     console.log();
@@ -59,10 +60,10 @@ async function main() {
         }
     }
     const sInstall = spinner();
-    sInstall.start("Installing packages and fetching documentation...");
+    sInstall.start("Installing packages...");
     for (const pkg of packagesToInstall) {
         try {
-            await installPackagesAndFetchDocs([pkg], { silent: true });
+            await installPackages([pkg], { silent: true });
         }
         catch (error) {
             sInstall.stop(`❌ Failed to install ${pkg}.`);
@@ -70,11 +71,15 @@ async function main() {
             process.exit(1);
         }
     }
-    sInstall.stop("✅ Packages installed and documentation fetched.");
+    sInstall.stop("✅ Package installation complete.");
     projectManifest.reload();
     projectManifest.updateTasks(getProjectTasks());
     const finalInstalledPackages = getInstalledPackageConfigs(projectManifest.getInstalledPackages()).map((pkg) => pkg.value);
-    const docsMapping = await syncPackageDocs(finalInstalledPackages, packagesToInstall);
+    const finalInstalledVersions = getInstalledPackageVersions(projectManifest.getInstalledPackageSpecifiers());
+    const sDocs = spinner();
+    sDocs.start("Fetching documentation for installed package versions...");
+    const docsMapping = await syncPackageDocs(finalInstalledPackages, finalInstalledVersions);
+    sDocs.stop("✅ Documentation synchronization complete.");
     createFolderStructure(finalInstalledPackages);
     ensureGitignore();
     ensureEnvFile();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nshiab/setup-data-project",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "Quickly kick off a project using simple-data-analysis-core, simple-data-analysis, and journalism libraries.",
   "repository": {
     "type": "git",

package/script/helpers/ensureAgents.d.ts

@@ -1,2 +1,3 @@
-export declare function ensureAgents(docsMapping: Record<string, string>, runtime: string, installedPackages?: string[]): void;
+import type { PackageDocs } from "./fetchPackageDocs.js";
+export declare function ensureAgents(docsMapping: Record<string, PackageDocs>, runtime: string, installedPackages?: string[]): void;
 //# sourceMappingURL=ensureAgents.d.ts.map

package/script/helpers/ensureAgents.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ensureAgents.d.ts","sourceRoot":"","sources":["../../src/helpers/ensureAgents.ts"],"names":[],"mappings":"AAOA,wBAAgB,YAAY,CAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACnC,OAAO,EAAE,MAAM,EACf,iBAAiB,GAAE,MAAM,EAA6B,QAmJvD"}
+{"version":3,"file":"ensureAgents.d.ts","sourceRoot":"","sources":["../../src/helpers/ensureAgents.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAOzD,wBAAgB,YAAY,CAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACxC,OAAO,EAAE,MAAM,EACf,iBAAiB,GAAE,MAAM,EAA6B,QAqLvD"}

package/script/helpers/ensureAgents.js

@@ -8,11 +8,15 @@ function ensureAgents(docsMapping, runtime, installedPackages = Object.keys(docs
     let journalismFunctions = "";
     let sdaClassesAndMethods = "";
     let sdaDocsPath = "";
+    let sdaReadmePath = "";
     let sdaLibraryName = "";
     const journalismFunctionsByPackage = new Map();
-    for (const [pkg, doc] of Object.entries(docsMapping)) {
+    for (const [pkg, docs] of Object.entries(docsMapping)) {
         const pkgConfig = packageRegistry_js_1.SUPPORTED_PACKAGES.find((p) => p.value === pkg);
         const pkgType = pkgConfig?.type || "other";
+        const doc = docs.llm;
+        if (doc === undefined)
+            continue;
         if (pkgType === "journalism") {
             const repoName = pkg.split("/")[1];
             const functions = doc
@@ -30,7 +34,10 @@ function ensureAgents(docsMapping, runtime, installedPackages = Object.keys(docs
         }
         else if (pkgType === "sda") {
             const repoName = pkg.split("/")[1];
-            sdaDocsPath = `./docs/${repoName}.md`;
+            sdaDocsPath = `./docs/${repoName}/llm.md`;
+            sdaReadmePath = docs.readme === undefined
+                ? ""
+                : `./docs/${repoName}/README.md`;
             sdaLibraryName = pkgConfig?.label || repoName;
             sdaClassesAndMethods = doc
                 .split("\n")
@@ -92,6 +99,30 @@ function ensureAgents(docsMapping, runtime, installedPackages = Object.keys(docs
         libraryGuidance = `
 Always prioritize the installed ${libraryNames} ${libraryNoun} when relevant.
 `;
+        const documentationGuidance = installedPackageConfigs.flatMap((pkg) => {
+            const docs = docsMapping[pkg.value];
+            if (!docs)
+                return [];
+            const repoName = pkg.value.split("/")[1];
+            const availableDocs = [
+                docs.readme === undefined
+                    ? ""
+                    : `overview and practical examples at \`./docs/${repoName}/README.md\``,
+                docs.llm === undefined
+                    ? ""
+                    : `complete API documentation at \`./docs/${repoName}/llm.md\``,
+            ].filter(Boolean);
+            return availableDocs.length === 0
+                ? []
+                : [`- \`${pkg.value}\`: ${availableDocs.join("; ")}`];
+        });
+        if (documentationGuidance.length > 0) {
+            libraryGuidance += `
+The following documentation was fetched for the exact installed library versions. Consult it when relevant:
+
+${documentationGuidance.join("\n")}
+`;
+        }
         if (importExamples.length > 0) {
             libraryGuidance += `APIs can be imported with named imports like this:
 \`\`\`typescript
@@ -120,12 +151,15 @@ If you need to output data to a file, always put the file in the "sda/output" fo
 ${libraryGuidance}`;
     if (journalismFunctions !== "") {
         content += `
-Here are the functions available in the "journalism" libraries. If one of the function might be relevant, read the complete documentation in the "./docs/" folder to properly use it.
+Here are the functions available in the "journalism" libraries. If one of the functions might be relevant, search its complete API documentation in the corresponding \`./docs/<library-name>/llm.md\` file to properly use it.
 ${journalismFunctions}`;
     }
     if (sdaClassesAndMethods !== "") {
+        const readmeGuidance = sdaReadmePath === ""
+            ? ""
+            : ` An overview and practical examples are available at "${sdaReadmePath}".`;
         content += `
-Here are the classes and their methods available in the "${sdaLibraryName}" library. If one of the classes or methods might be relevant, read the complete documentation at "${sdaDocsPath}" to properly use it.
+Here are the classes and their methods available in the "${sdaLibraryName}" library.${readmeGuidance} If one of the classes or methods might be relevant, search the complete API documentation at "${sdaDocsPath}" to properly use it.
 
 Most data-loading and transformation methods are synchronous, chainable builders. Await the final asynchronous observer or export method, such as \`log()\`, \`getData()\`, or \`writeData()\`, to execute the queued operations. If a chain ends without an observer, call \`run()\`. Always call \`await sdb.close()\` when the database is no longer needed. Methods that return an answer or export to an external service can be asynchronous, so verify their documentation before calling them.
 ${sdaClassesAndMethods}`;

package/script/helpers/fetchPackageDocs.d.ts

@@ -0,0 +1,8 @@
+export interface PackageDocs {
+    readme?: string;
+    llm?: string;
+}
+export declare function fetchPackageDocs(pkg: string, version: string, options?: {
+    silent?: boolean;
+}): Promise<PackageDocs>;
+//# sourceMappingURL=fetchPackageDocs.d.ts.map

package/script/helpers/fetchPackageDocs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetchPackageDocs.d.ts","sourceRoot":"","sources":["../../src/helpers/fetchPackageDocs.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAiBD,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAO,GACjC,OAAO,CAAC,WAAW,CAAC,CA4DtB"}