@nshiab/setup-data-project 2.0.3 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/README.md +76 -28
  2. package/dist/main.mjs +849 -0
  3. package/dist/main.mjs.map +1 -0
  4. package/package.json +29 -18
  5. package/esm/_dnt.shims.d.ts +0 -2
  6. package/esm/_dnt.shims.d.ts.map +0 -1
  7. package/esm/_dnt.shims.js +0 -57
  8. package/esm/helpers/combinePlotDocs.d.ts +0 -2
  9. package/esm/helpers/combinePlotDocs.d.ts.map +0 -1
  10. package/esm/helpers/combinePlotDocs.js +0 -111
  11. package/esm/helpers/createFolderStructure.d.ts +0 -2
  12. package/esm/helpers/createFolderStructure.d.ts.map +0 -1
  13. package/esm/helpers/createFolderStructure.js +0 -40
  14. package/esm/helpers/ensureAgents.d.ts +0 -3
  15. package/esm/helpers/ensureAgents.d.ts.map +0 -1
  16. package/esm/helpers/ensureAgents.js +0 -186
  17. package/esm/helpers/ensureEnvFile.d.ts +0 -2
  18. package/esm/helpers/ensureEnvFile.d.ts.map +0 -1
  19. package/esm/helpers/ensureEnvFile.js +0 -8
  20. package/esm/helpers/ensureGitignore.d.ts +0 -3
  21. package/esm/helpers/ensureGitignore.d.ts.map +0 -1
  22. package/esm/helpers/ensureGitignore.js +0 -24
  23. package/esm/helpers/ensureManagedSection.d.ts +0 -9
  24. package/esm/helpers/ensureManagedSection.d.ts.map +0 -1
  25. package/esm/helpers/ensureManagedSection.js +0 -33
  26. package/esm/helpers/ensureReadme.d.ts +0 -2
  27. package/esm/helpers/ensureReadme.d.ts.map +0 -1
  28. package/esm/helpers/ensureReadme.js +0 -61
  29. package/esm/helpers/fetchPackageDocs.d.ts +0 -12
  30. package/esm/helpers/fetchPackageDocs.d.ts.map +0 -1
  31. package/esm/helpers/fetchPackageDocs.js +0 -75
  32. package/esm/helpers/fetchPlotDocs.d.ts +0 -5
  33. package/esm/helpers/fetchPlotDocs.d.ts.map +0 -1
  34. package/esm/helpers/fetchPlotDocs.js +0 -103
  35. package/esm/helpers/getInstalledPackageVersions.d.ts +0 -2
  36. package/esm/helpers/getInstalledPackageVersions.d.ts.map +0 -1
  37. package/esm/helpers/getInstalledPackageVersions.js +0 -94
  38. package/esm/helpers/getProjectTasks.d.ts +0 -6
  39. package/esm/helpers/getProjectTasks.d.ts.map +0 -1
  40. package/esm/helpers/getProjectTasks.js +0 -17
  41. package/esm/helpers/getRuntime.d.ts +0 -5
  42. package/esm/helpers/getRuntime.d.ts.map +0 -1
  43. package/esm/helpers/getRuntime.js +0 -19
  44. package/esm/helpers/gitignoreEntries.d.ts +0 -2
  45. package/esm/helpers/gitignoreEntries.d.ts.map +0 -1
  46. package/esm/helpers/gitignoreEntries.js +0 -10
  47. package/esm/helpers/installPackages.d.ts +0 -7
  48. package/esm/helpers/installPackages.d.ts.map +0 -1
  49. package/esm/helpers/installPackages.js +0 -49
  50. package/esm/helpers/packageOptions.d.ts +0 -6
  51. package/esm/helpers/packageOptions.d.ts.map +0 -1
  52. package/esm/helpers/packageOptions.js +0 -6
  53. package/esm/helpers/packageRegistry.d.ts +0 -10
  54. package/esm/helpers/packageRegistry.d.ts.map +0 -1
  55. package/esm/helpers/packageRegistry.js +0 -89
  56. package/esm/helpers/parseArguments.d.ts +0 -7
  57. package/esm/helpers/parseArguments.d.ts.map +0 -1
  58. package/esm/helpers/parseArguments.js +0 -30
  59. package/esm/helpers/plotDocPages.d.ts +0 -2
  60. package/esm/helpers/plotDocPages.d.ts.map +0 -1
  61. package/esm/helpers/plotDocPages.js +0 -46
  62. package/esm/helpers/projectManifest.d.ts +0 -12
  63. package/esm/helpers/projectManifest.d.ts.map +0 -1
  64. package/esm/helpers/projectManifest.js +0 -80
  65. package/esm/helpers/syncPackageDocs.d.ts +0 -3
  66. package/esm/helpers/syncPackageDocs.d.ts.map +0 -1
  67. package/esm/helpers/syncPackageDocs.js +0 -50
  68. package/esm/main.d.ts +0 -3
  69. package/esm/main.d.ts.map +0 -1
  70. package/esm/main.js +0 -104
  71. package/esm/package.json +0 -3
  72. package/script/_dnt.shims.d.ts +0 -2
  73. package/script/_dnt.shims.d.ts.map +0 -1
  74. package/script/_dnt.shims.js +0 -60
  75. package/script/helpers/combinePlotDocs.d.ts +0 -2
  76. package/script/helpers/combinePlotDocs.d.ts.map +0 -1
  77. package/script/helpers/combinePlotDocs.js +0 -114
  78. package/script/helpers/createFolderStructure.d.ts +0 -2
  79. package/script/helpers/createFolderStructure.d.ts.map +0 -1
  80. package/script/helpers/createFolderStructure.js +0 -43
  81. package/script/helpers/ensureAgents.d.ts +0 -3
  82. package/script/helpers/ensureAgents.d.ts.map +0 -1
  83. package/script/helpers/ensureAgents.js +0 -189
  84. package/script/helpers/ensureEnvFile.d.ts +0 -2
  85. package/script/helpers/ensureEnvFile.d.ts.map +0 -1
  86. package/script/helpers/ensureEnvFile.js +0 -11
  87. package/script/helpers/ensureGitignore.d.ts +0 -3
  88. package/script/helpers/ensureGitignore.d.ts.map +0 -1
  89. package/script/helpers/ensureGitignore.js +0 -28
  90. package/script/helpers/ensureManagedSection.d.ts +0 -9
  91. package/script/helpers/ensureManagedSection.d.ts.map +0 -1
  92. package/script/helpers/ensureManagedSection.js +0 -36
  93. package/script/helpers/ensureReadme.d.ts +0 -2
  94. package/script/helpers/ensureReadme.d.ts.map +0 -1
  95. package/script/helpers/ensureReadme.js +0 -64
  96. package/script/helpers/fetchPackageDocs.d.ts +0 -12
  97. package/script/helpers/fetchPackageDocs.d.ts.map +0 -1
  98. package/script/helpers/fetchPackageDocs.js +0 -78
  99. package/script/helpers/fetchPlotDocs.d.ts +0 -5
  100. package/script/helpers/fetchPlotDocs.d.ts.map +0 -1
  101. package/script/helpers/fetchPlotDocs.js +0 -106
  102. package/script/helpers/getInstalledPackageVersions.d.ts +0 -2
  103. package/script/helpers/getInstalledPackageVersions.d.ts.map +0 -1
  104. package/script/helpers/getInstalledPackageVersions.js +0 -97
  105. package/script/helpers/getProjectTasks.d.ts +0 -6
  106. package/script/helpers/getProjectTasks.d.ts.map +0 -1
  107. package/script/helpers/getProjectTasks.js +0 -20
  108. package/script/helpers/getRuntime.d.ts +0 -5
  109. package/script/helpers/getRuntime.d.ts.map +0 -1
  110. package/script/helpers/getRuntime.js +0 -59
  111. package/script/helpers/gitignoreEntries.d.ts +0 -2
  112. package/script/helpers/gitignoreEntries.d.ts.map +0 -1
  113. package/script/helpers/gitignoreEntries.js +0 -13
  114. package/script/helpers/installPackages.d.ts +0 -7
  115. package/script/helpers/installPackages.d.ts.map +0 -1
  116. package/script/helpers/installPackages.js +0 -86
  117. package/script/helpers/packageOptions.d.ts +0 -6
  118. package/script/helpers/packageOptions.d.ts.map +0 -1
  119. package/script/helpers/packageOptions.js +0 -9
  120. package/script/helpers/packageRegistry.d.ts +0 -10
  121. package/script/helpers/packageRegistry.d.ts.map +0 -1
  122. package/script/helpers/packageRegistry.js +0 -93
  123. package/script/helpers/parseArguments.d.ts +0 -7
  124. package/script/helpers/parseArguments.d.ts.map +0 -1
  125. package/script/helpers/parseArguments.js +0 -34
  126. package/script/helpers/plotDocPages.d.ts +0 -2
  127. package/script/helpers/plotDocPages.d.ts.map +0 -1
  128. package/script/helpers/plotDocPages.js +0 -49
  129. package/script/helpers/projectManifest.d.ts +0 -12
  130. package/script/helpers/projectManifest.d.ts.map +0 -1
  131. package/script/helpers/projectManifest.js +0 -84
  132. package/script/helpers/syncPackageDocs.d.ts +0 -3
  133. package/script/helpers/syncPackageDocs.d.ts.map +0 -1
  134. package/script/helpers/syncPackageDocs.js +0 -53
  135. package/script/main.d.ts +0 -3
  136. package/script/main.d.ts.map +0 -1
  137. package/script/main.js +0 -109
  138. package/script/package.json +0 -3
@@ -1,186 +0,0 @@
1
- import { log } from "@clack/prompts";
2
- import { ensureManagedSection } from "./ensureManagedSection.js";
3
- import { getInstalledPackageConfigs, SUPPORTED_PACKAGES, } from "./packageRegistry.js";
4
- export function ensureAgents(docsMapping, runtime, installedPackages = Object.keys(docsMapping)) {
5
- let journalismFunctions = "";
6
- let sdaClassesAndMethods = "";
7
- let sdaDocsPath = "";
8
- let sdaReadmePath = "";
9
- let sdaLibraryName = "";
10
- const journalismFunctionsByPackage = new Map();
11
- for (const [pkg, docs] of Object.entries(docsMapping)) {
12
- const pkgConfig = SUPPORTED_PACKAGES.find((p) => p.value === pkg);
13
- const pkgType = pkgConfig?.type || "other";
14
- const doc = docs.llm;
15
- if (doc === undefined)
16
- continue;
17
- if (pkgType === "journalism") {
18
- const repoName = pkg.split("/")[1];
19
- const functions = doc
20
- .split("\n")
21
- .filter((line) => line.trim().startsWith("#"))
22
- .filter((line) => {
23
- const depth = line.match(/^#+/)?.[0].length || 0;
24
- return depth === 2;
25
- })
26
- .map((line) => line.replace(/^#+\s+/, "").trim());
27
- journalismFunctionsByPackage.set(pkg, functions);
28
- journalismFunctions += `\n### ${repoName}\n\n`;
29
- journalismFunctions += functions.map((name) => `- ${name}`).join("\n") +
30
- "\n";
31
- }
32
- else if (pkgType === "sda") {
33
- const repoName = pkg.split("/")[1];
34
- sdaDocsPath = `./docs/${repoName}/llm.md`;
35
- sdaReadmePath = docs.readme === undefined
36
- ? ""
37
- : `./docs/${repoName}/README.md`;
38
- sdaLibraryName = pkgConfig?.label || repoName;
39
- sdaClassesAndMethods = doc
40
- .split("\n")
41
- .filter((line) => line.trim().startsWith("#"))
42
- .filter((line) => {
43
- const depth = line.match(/^#+/)?.[0].length || 0;
44
- if (depth === 2)
45
- return true;
46
- if (depth !== 4)
47
- return false;
48
- const heading = line.replace(/^#+\s+/, "").replaceAll("`", "")
49
- .trim();
50
- return heading !== "Parameters";
51
- })
52
- .map((line) => {
53
- const depth = line.match(/^#+/)?.[0].length || 0;
54
- if (depth === 2) {
55
- return "\n" + line.replace(/^#+\s+/, "").trim();
56
- }
57
- else {
58
- return line.replace(/^#+\s+/, " - ")
59
- .replaceAll("`", "");
60
- }
61
- })
62
- .join("\n");
63
- }
64
- }
65
- const configFile = runtime === "deno" ? "deno.json" : "package.json";
66
- const runSdaOnce = runtime === "deno"
67
- ? "deno run -A --env --check sda/main.ts"
68
- : runtime === "bun"
69
- ? "bun run sda/main.ts"
70
- : "node --env-file-if-exists=.env --experimental-strip-types sda/main.ts";
71
- const runClean = runtime === "deno" ? "deno task clean" : "npm run clean";
72
- const runAllTests = runtime === "deno"
73
- ? "deno task all-tests"
74
- : runtime === "bun"
75
- ? "bun run all-tests"
76
- : "npm run all-tests";
77
- const installedPackageConfigs = getInstalledPackageConfigs(installedPackages);
78
- const importExamples = installedPackageConfigs.flatMap((pkg) => {
79
- if (pkg.type === "sda") {
80
- return [`import { SimpleDB } from "${pkg.value}";`];
81
- }
82
- const functionName = journalismFunctionsByPackage.get(pkg.value)
83
- ?.map((name) => name.replaceAll("`", ""))
84
- .find((name) => /^[A-Za-z_$][\w$]*$/.test(name));
85
- return functionName
86
- ? [`import { ${functionName} } from "${pkg.value}";`]
87
- : [];
88
- });
89
- let libraryGuidance = "";
90
- if (installedPackageConfigs.length > 0) {
91
- const libraryNames = installedPackageConfigs.map((pkg) => `"${pkg.value}"`)
92
- .join(", ");
93
- const libraryNoun = installedPackageConfigs.length === 1
94
- ? "library"
95
- : "libraries";
96
- libraryGuidance = `
97
- Always prioritize the installed ${libraryNames} ${libraryNoun} when relevant.
98
- `;
99
- const documentationGuidance = installedPackageConfigs.flatMap((pkg) => {
100
- const docs = docsMapping[pkg.value];
101
- if (!docs)
102
- return [];
103
- if (docs.plot) {
104
- return [
105
- "- `@observablehq/plot`: use Observable Plot to create charts and maps " +
106
- "with `writeChart` and `writeMap`. Import with " +
107
- '`import { plot } from "@observablehq/plot";`. ' +
108
- "Documentation and examples at `./docs/observable-plot/llm.md`",
109
- ];
110
- }
111
- const repoName = pkg.value.split("/")[1];
112
- const availableDocs = [
113
- docs.readme === undefined
114
- ? ""
115
- : `overview and practical examples at \`./docs/${repoName}/README.md\``,
116
- docs.llm === undefined
117
- ? ""
118
- : `complete API documentation at \`./docs/${repoName}/llm.md\``,
119
- ].filter(Boolean);
120
- return availableDocs.length === 0
121
- ? []
122
- : [`- \`${pkg.value}\`: ${availableDocs.join("; ")}`];
123
- });
124
- if (documentationGuidance.length > 0) {
125
- libraryGuidance += `
126
- The \`docs/\` folder contains long, exhaustive library references. Search for the relevant methods or topics, then read only the matching sections and any related definitions or examples you need.
127
-
128
- The following documentation was fetched for the exact installed library versions. Consult it when relevant:
129
-
130
- ${documentationGuidance.join("\n")}
131
- `;
132
- }
133
- if (importExamples.length > 0) {
134
- libraryGuidance += `APIs can be imported like this:
135
- \`\`\`typescript
136
- ${importExamples.join("\n")}
137
- \`\`\`
138
- `;
139
- }
140
- }
141
- let content = `Always verify if there is a ${configFile} file in the root of the project and familiarize yourself with the scripts available in it and the libraries already installed in the project.
142
-
143
- Run \`sda/main.ts\` with \`${runSdaOnce}\`. Do not use the \`sda\` task for agent-driven checks because it runs in watch mode and will keep waiting for file changes.
144
-
145
- Before handing off your work, always run \`${runAllTests}\`, even if no tests have been added yet. Fix any errors or warnings it reports.
146
-
147
- The \`clean\` task (e.g. \`${runClean}\`) removes caches and other temporary files. Do not run it routinely. Cached results can be especially valuable for computationally expensive operations or API calls. If you are unsure whether the caches should be removed, ask the user first.
148
-
149
- Always use "sda/main.ts" as the entry point.
150
-
151
- If you need to create other TypeScript files, create them in the "sda/helpers" folder. Prioritize the use of helper functions to keep the code well organized and maintainable, with one helper function per file, with the file named after the function. Prioritize default exports for helper functions.
152
-
153
- Put tests in the "sda/tests" folder. Write focused tests for reusable helpers and important data transformations when appropriate. When testing a helper from "sda/helpers/functionName.ts", name its test file "sda/tests/functionName.test.ts".
154
-
155
- If you need to download data, always put the files in the "sda/data" folder, which is ignored by Git.
156
-
157
- If you need to output data to a file, always put the file in the "sda/output" folder, which is also ignored by Git.
158
- ${libraryGuidance}`;
159
- if (journalismFunctions !== "") {
160
- content += `
161
- Here are the functions available in the "journalism" libraries. If one of the functions might be relevant, search its complete API documentation in the corresponding \`./docs/<library-name>/llm.md\` file to properly use it.
162
- ${journalismFunctions}`;
163
- }
164
- if (sdaClassesAndMethods !== "") {
165
- const readmeGuidance = sdaReadmePath === ""
166
- ? ""
167
- : ` An overview and practical examples are available at "${sdaReadmePath}".`;
168
- content += `
169
- Here are the classes and their methods available in the "${sdaLibraryName}" library.${readmeGuidance} If one of the classes or methods might be relevant, search the complete API documentation at "${sdaDocsPath}" to properly use it.
170
-
171
- Most data-loading and transformation methods are synchronous, chainable builders. Await the final asynchronous observer or export method, such as \`log()\`, \`getData()\`, or \`writeData()\`, to execute the queued operations. If a chain ends without an observer, call \`run()\`. Always call \`await sdb.close()\` when the database is no longer needed. Methods that return an answer or export to an external service can be asynchronous, so verify their documentation before calling them.
172
- ${sdaClassesAndMethods}`;
173
- }
174
- const status = ensureManagedSection({
175
- path: "AGENTS.md",
176
- name: "agents",
177
- content,
178
- createContent: (managedSection) => managedSection,
179
- });
180
- if (status === "created") {
181
- log.info("Created AGENTS.md");
182
- }
183
- else if (status === "updated") {
184
- log.info("Updated AGENTS.md");
185
- }
186
- }
@@ -1,2 +0,0 @@
1
- export declare function ensureEnvFile(): void;
2
- //# sourceMappingURL=ensureEnvFile.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"ensureEnvFile.d.ts","sourceRoot":"","sources":["../../src/helpers/ensureEnvFile.ts"],"names":[],"mappings":"AAGA,wBAAgB,aAAa,SAK5B"}
@@ -1,8 +0,0 @@
1
- import { lstatSync, writeFileSync } from "node:fs";
2
- import { log } from "@clack/prompts";
3
- export function ensureEnvFile() {
4
- if (lstatSync(".env", { throwIfNoEntry: false }) === undefined) {
5
- writeFileSync(".env", "");
6
- log.info("Created .env");
7
- }
8
- }
@@ -1,3 +0,0 @@
1
- export declare const GITIGNORE_SENTINEL = "# Added by setup-data-project. Do not remove this line.";
2
- export declare function ensureGitignore(): void;
3
- //# sourceMappingURL=ensureGitignore.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"ensureGitignore.d.ts","sourceRoot":"","sources":["../../src/helpers/ensureGitignore.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,kBAAkB,4DAC4B,CAAC;AAE5D,wBAAgB,eAAe,SA0B9B"}
@@ -1,24 +0,0 @@
1
- import { lstatSync, readFileSync, writeFileSync } from "node:fs";
2
- import { log } from "@clack/prompts";
3
- import { GITIGNORE_ENTRIES } from "./gitignoreEntries.js";
4
- export const GITIGNORE_SENTINEL = "# Added by setup-data-project. Do not remove this line.";
5
- export function ensureGitignore() {
6
- const path = ".gitignore";
7
- const stats = lstatSync(path, { throwIfNoEntry: false });
8
- if (stats?.isSymbolicLink()) {
9
- return;
10
- }
11
- const content = stats === undefined ? "" : readFileSync(path, "utf-8");
12
- const hasSentinel = content.split("\n").some((line) => line.trim() === GITIGNORE_SENTINEL);
13
- if (hasSentinel) {
14
- return;
15
- }
16
- const separator = content === ""
17
- ? ""
18
- : content.endsWith("\n")
19
- ? "\n"
20
- : "\n\n";
21
- const block = `${GITIGNORE_SENTINEL}\n${GITIGNORE_ENTRIES.join("\n")}\n`;
22
- writeFileSync(path, content + separator + block);
23
- log.info(`${stats === undefined ? "Created" : "Updated"} ${path}`);
24
- }
@@ -1,9 +0,0 @@
1
- interface ManagedSectionOptions {
2
- path: string;
3
- name: string;
4
- content: string;
5
- createContent: (section: string) => string;
6
- }
7
- export declare function ensureManagedSection(options: ManagedSectionOptions): "created" | "updated" | "unchanged" | "missing-markers";
8
- export {};
9
- //# sourceMappingURL=ensureManagedSection.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"ensureManagedSection.d.ts","sourceRoot":"","sources":["../../src/helpers/ensureManagedSection.ts"],"names":[],"mappings":"AAGA,UAAU,qBAAqB;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC;CAC5C;AAED,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,qBAAqB,GAC7B,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,iBAAiB,CAwCzD"}
@@ -1,33 +0,0 @@
1
- import { log } from "@clack/prompts";
2
- import { existsSync, readFileSync, writeFileSync } from "node:fs";
3
- export function ensureManagedSection(options) {
4
- const startMarker = `<!-- Do not remove / setup-data-project:${options.name}:start -->`;
5
- const endMarker = `<!-- Do not remove / setup-data-project:${options.name}:end -->`;
6
- const trimmedContent = options.content.trim();
7
- const section = trimmedContent === ""
8
- ? `${startMarker}\n${endMarker}`
9
- : `${startMarker}\n${trimmedContent}\n${endMarker}`;
10
- if (!existsSync(options.path)) {
11
- writeFileSync(options.path, ensureFinalNewline(options.createContent(section)));
12
- return "created";
13
- }
14
- const currentContent = readFileSync(options.path, "utf-8");
15
- const start = currentContent.indexOf(startMarker);
16
- const end = currentContent.indexOf(endMarker);
17
- if (start === -1 || end === -1 || end < start) {
18
- log.warn(`${options.path} was not updated because its setup-data-project markers ` +
19
- `were not found. Add ${startMarker} and ${endMarker}, then rerun the ` +
20
- "script.");
21
- return "missing-markers";
22
- }
23
- const nextContent = currentContent.slice(0, start) + section +
24
- currentContent.slice(end + endMarker.length);
25
- if (nextContent === currentContent) {
26
- return "unchanged";
27
- }
28
- writeFileSync(options.path, ensureFinalNewline(nextContent));
29
- return "updated";
30
- }
31
- function ensureFinalNewline(content) {
32
- return content.endsWith("\n") ? content : content + "\n";
33
- }
@@ -1,2 +0,0 @@
1
- export declare function ensureReadme(runtime: string, installedPackages?: string[]): void;
2
- //# sourceMappingURL=ensureReadme.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"ensureReadme.d.ts","sourceRoot":"","sources":["../../src/helpers/ensureReadme.ts"],"names":[],"mappings":"AAIA,wBAAgB,YAAY,CAC1B,OAAO,EAAE,MAAM,EACf,iBAAiB,GAAE,MAAM,EAAO,QAsEjC"}
@@ -1,61 +0,0 @@
1
- import { log } from "@clack/prompts";
2
- import { ensureManagedSection } from "./ensureManagedSection.js";
3
- import { getInstalledPackageConfigs } from "./packageRegistry.js";
4
- export function ensureReadme(runtime, installedPackages = []) {
5
- let runCommand = "npm run";
6
- if (runtime === "deno") {
7
- runCommand = "deno task";
8
- }
9
- else if (runtime === "bun") {
10
- runCommand = "bun run";
11
- }
12
- const installedPackageConfigs = getInstalledPackageConfigs(installedPackages);
13
- const hasSdaCore = installedPackageConfigs.some((pkg) => pkg.value === "@nshiab/simple-data-analysis-core");
14
- const hasFullSda = installedPackageConfigs.some((pkg) => pkg.value === "@nshiab/simple-data-analysis");
15
- const hasJournalism = installedPackageConfigs.some((pkg) => pkg.type === "journalism");
16
- const libraryLinks = [
17
- hasSdaCore
18
- ? "[simple-data-analysis-core](https://github.com/nshiab/simple-data-analysis-core)"
19
- : "",
20
- hasFullSda
21
- ? "[simple-data-analysis](https://github.com/nshiab/simple-data-analysis)"
22
- : "",
23
- hasJournalism ? "[journalism](https://github.com/nshiab/journalism)" : "",
24
- ].filter(Boolean);
25
- const librarySection = libraryLinks.length === 0
26
- ? ""
27
- : `It has installed ${libraryLinks.join(" and ")} ${libraryLinks.length === 1 ? "library" : "libraries"}, along with
28
- up-to-date documentation and AI agent instructions.`;
29
- const status = ensureManagedSection({
30
- path: "README.md",
31
- name: "libraries",
32
- content: librarySection,
33
- createContent: (managedSection) => `This repository was created with
34
- [setup-data-project](https://github.com/nshiab/setup-data-project/).
35
-
36
- ${managedSection}
37
-
38
- Here's the recommended workflow:
39
-
40
- - Put your raw data in the \`sda/data\` folder. Note that this folder is
41
- ignored by Git.
42
- - Use the \`sda/main.ts\` file to process your raw data. Write the
43
- results to the \`sda/output\` folder, which is also ignored by Git.
44
- - Put your tests in the \`sda/tests\` folder.
45
-
46
- When working on your project, use the following commands:
47
-
48
- - \`${runCommand} sda\` will watch your \`sda/main.ts\` and its dependencies.
49
- Whenever you save changes, the data will be reprocessed.
50
- - \`${runCommand} all-tests\` will run the project's test command.
51
- - \`${runCommand} clean\` will remove cache and temporary files, if present.
52
-
53
- Have fun!`,
54
- });
55
- if (status === "created") {
56
- log.info("Created README.md");
57
- }
58
- else if (status === "updated") {
59
- log.info("Updated README.md");
60
- }
61
- }
@@ -1,12 +0,0 @@
1
- export interface PackageDocs {
2
- readme?: string;
3
- llm?: string;
4
- plot?: {
5
- version: string;
6
- pages: string[];
7
- };
8
- }
9
- export declare function fetchPackageDocs(pkg: string, version: string, options?: {
10
- silent?: boolean;
11
- }): Promise<PackageDocs>;
12
- //# sourceMappingURL=fetchPackageDocs.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"fetchPackageDocs.d.ts","sourceRoot":"","sources":["../../src/helpers/fetchPackageDocs.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;CAC7C;AAiBD,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAO,GACjC,OAAO,CAAC,WAAW,CAAC,CA4DtB"}
@@ -1,75 +0,0 @@
1
- import { log, spinner } from "@clack/prompts";
2
- import { existsSync, mkdirSync, writeFileSync } from "node:fs";
3
- import { join } from "node:path";
4
- const PACKAGE_DOCUMENTS = [
5
- {
6
- remoteName: "README.md",
7
- localName: "README.md",
8
- key: "readme",
9
- label: "README",
10
- },
11
- {
12
- remoteName: "llm.md",
13
- localName: "llm.md",
14
- key: "llm",
15
- label: "API documentation",
16
- },
17
- ];
18
- export async function fetchPackageDocs(pkg, version, options = {}) {
19
- const repoName = pkg.split("/")[1];
20
- const docsDirectory = join("docs", repoName);
21
- mkdirSync(docsDirectory, { recursive: true });
22
- const sFetch = spinner();
23
- if (!options.silent) {
24
- sFetch.start(`Fetching documentation for ${pkg}@${version}...`);
25
- }
26
- const fetchedDocuments = await Promise.all(PACKAGE_DOCUMENTS.map(async (document) => {
27
- const localPath = join(docsDirectory, document.localName);
28
- try {
29
- const url = "https://raw.githubusercontent.com/nshiab/" + repoName +
30
- `/refs/tags/v${version}/${document.remoteName}`;
31
- const response = await fetch(url);
32
- if (!response.ok) {
33
- warnAboutFailedFetch(pkg, version, document.label, localPath);
34
- return { key: document.key, content: undefined };
35
- }
36
- const content = await response.text();
37
- writeFileSync(localPath, content);
38
- return { key: document.key, content };
39
- }
40
- catch (error) {
41
- warnAboutFailedFetch(pkg, version, document.label, localPath);
42
- console.error(error);
43
- return { key: document.key, content: undefined };
44
- }
45
- }));
46
- const docs = {};
47
- for (const document of fetchedDocuments) {
48
- if (document.content !== undefined) {
49
- docs[document.key] = document.content;
50
- }
51
- }
52
- const savedCount = fetchedDocuments.filter((document) => document.content !== undefined)
53
- .length;
54
- if (!options.silent) {
55
- if (savedCount === PACKAGE_DOCUMENTS.length) {
56
- sFetch.stop(`✅ Documentation for ${pkg}@${version} saved!`);
57
- }
58
- else if (savedCount > 0) {
59
- sFetch.stop(`⚠️ Some documentation for ${pkg}@${version} could not be fetched.`);
60
- }
61
- else {
62
- sFetch.stop(`❌ Failed to fetch documentation for ${pkg}@${version}.`);
63
- }
64
- }
65
- return docs;
66
- }
67
- function warnAboutFailedFetch(pkg, version, label, localPath) {
68
- const existingFileGuidance = existsSync(localPath)
69
- ? ` The existing file at ${localPath} was preserved, but it may not ` +
70
- "match the installed version. Double-check it before relying on it, or " +
71
- "remove it."
72
- : "";
73
- log.warn(`Could not fetch ${label} for ${pkg}@${version} from its GitHub tag.` +
74
- existingFileGuidance);
75
- }
@@ -1,5 +0,0 @@
1
- import type { PackageDocs } from "./fetchPackageDocs.js";
2
- export declare function fetchPlotDocs(version: string, options?: {
3
- preferStored?: boolean;
4
- }): Promise<PackageDocs>;
5
- //# sourceMappingURL=fetchPlotDocs.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"fetchPlotDocs.d.ts","sourceRoot":"","sources":["../../src/helpers/fetchPlotDocs.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAGzD,wBAAsB,aAAa,CACjC,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;IAAE,YAAY,CAAC,EAAE,OAAO,CAAA;CAAO,GACvC,OAAO,CAAC,WAAW,CAAC,CAsCtB"}
@@ -1,103 +0,0 @@
1
- import { log } from "@clack/prompts";
2
- import { mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
3
- import { join } from "node:path";
4
- import { combinePlotDocs } from "./combinePlotDocs.js";
5
- import { PLOT_DOC_PAGES } from "./plotDocPages.js";
6
- // Publish one reference only after every tagged page has been fetched.
7
- export async function fetchPlotDocs(version, options = {}) {
8
- const tag = `v${version}`;
9
- const directory = join("docs", "observable-plot");
10
- try {
11
- let llm;
12
- if (options.preferStored !== false) {
13
- try {
14
- const response = await fetch("https://raw.githubusercontent.com/nshiab/setup-data-project/" +
15
- `refs/heads/main/docs-cache/observable-plot/${version}/llm.md`);
16
- if (response.ok) {
17
- const cached = await response.text();
18
- if (isStoredReference(cached, version))
19
- llm = cached;
20
- }
21
- }
22
- catch {
23
- // A cache is an optimization. An unavailable cache never blocks the tag.
24
- }
25
- }
26
- if (llm === undefined)
27
- llm = await generateReference(version);
28
- mkdirSync(directory, { recursive: true });
29
- const temporaryPath = join(directory, ".llm.md.tmp");
30
- try {
31
- writeFileSync(temporaryPath, llm);
32
- renameSync(temporaryPath, join(directory, "llm.md"));
33
- }
34
- finally {
35
- rmSync(temporaryPath, { force: true });
36
- }
37
- return { llm, plot: { version, pages: [...PLOT_DOC_PAGES] } };
38
- }
39
- catch (error) {
40
- log.warn(`Could not fetch complete Observable Plot documentation for ${tag}. ` +
41
- "Existing files were preserved, but may not match the installed " +
42
- "version. Plot documentation will not be included in AGENTS.md. " +
43
- String(error));
44
- return {};
45
- }
46
- }
47
- async function generateReference(version) {
48
- const tag = `v${version}`;
49
- const pages = PLOT_DOC_PAGES;
50
- const documents = new Map();
51
- const failed = [];
52
- // Fetch in small batches to avoid overwhelming GitHub on the first run.
53
- for (let offset = 0; offset < pages.length; offset += 8) {
54
- await Promise.all(pages.slice(offset, offset + 8).map(async (page) => {
55
- try {
56
- const url = "https://raw.githubusercontent.com/observablehq/plot/" +
57
- `refs/tags/${tag}/docs/${page}`;
58
- const response = await fetch(url);
59
- if (!response.ok)
60
- throw new Error(`HTTP ${response.status}`);
61
- const content = await response.text();
62
- documents.set(page, content);
63
- }
64
- catch {
65
- failed.push(page);
66
- }
67
- }));
68
- }
69
- if (failed.length > 0) {
70
- throw new Error(`Could not fetch ${failed.length} Markdown pages`);
71
- }
72
- return combinePlotDocs(version, documents);
73
- }
74
- function isStoredReference(content, version) {
75
- const source = `https://github.com/observablehq/plot/tree/v${version}/docs`;
76
- const metadata = content.split(/\n\s*\n/)[1]?.replace(/\s+/g, " ");
77
- if (!content.startsWith(`# Observable Plot ${version}\n\n`) ||
78
- metadata !==
79
- `Package: \`@observablehq/plot@${version}\`. Source: [v${version}](${source}).` ||
80
- !content.trimEnd().endsWith(`<!-- setup-data-project:observable-plot:${version}:complete -->`))
81
- return false;
82
- // Match actual page sections, not TOC entries. Require every selected page
83
- // exactly once and some upstream heading/content after its source record.
84
- const sections = [
85
- ...content.matchAll(/<a id="(plot-[^"]+)"><\/a>\n\n# ([^\n]+\.md)\n\nSource:\s+([^\s]+)\n\n/g),
86
- ];
87
- if (sections.length !== PLOT_DOC_PAGES.length)
88
- return false;
89
- const found = new Set();
90
- for (let index = 0; index < sections.length; index++) {
91
- const section = sections[index];
92
- const page = section[2];
93
- const id = `plot-${page.replace(/\.md$/, "").replaceAll("/", "--")}`;
94
- if (!PLOT_DOC_PAGES.includes(page) || found.has(page) || section[1] !== id ||
95
- section[3] !== `${source}/${page}`)
96
- return false;
97
- const body = content.slice(section.index + section[0].length, sections[index + 1]?.index);
98
- if (!/^#{1,6}\s+\S/m.test(body))
99
- return false;
100
- found.add(page);
101
- }
102
- return true;
103
- }
@@ -1,2 +0,0 @@
1
- export declare function getInstalledPackageVersions(packageSpecifiers: Record<string, string>): Record<string, string>;
2
- //# sourceMappingURL=getInstalledPackageVersions.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"getInstalledPackageVersions.d.ts","sourceRoot":"","sources":["../../src/helpers/getInstalledPackageVersions.ts"],"names":[],"mappings":"AAKA,wBAAgB,2BAA2B,CACzC,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACxC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CA0BxB"}