@adea-ai/plugins 4.0.1 → 4.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-plugins-catalog.d.ts +2 -4
- package/dist/agent-plugins-catalog.js +42 -50
- package/dist/agent-plugins-catalog.js.map +1 -1
- package/dist/agent-plugins-schema.d.ts +8 -0
- package/dist/agent-plugins-schema.js +10 -1
- package/dist/agent-plugins-schema.js.map +1 -1
- package/dist/artifacts.d.ts +57 -0
- package/dist/artifacts.js +159 -0
- package/dist/artifacts.js.map +1 -0
- package/dist/build.d.ts +20 -0
- package/dist/build.js +103 -0
- package/dist/build.js.map +1 -0
- package/dist/capabilities.d.ts +7 -0
- package/dist/capabilities.js +116 -0
- package/dist/capabilities.js.map +1 -0
- package/dist/catalog-store.d.ts +0 -5
- package/dist/catalog-store.js +1 -25
- package/dist/catalog-store.js.map +1 -1
- package/dist/concurrency.d.ts +1 -0
- package/dist/concurrency.js +15 -0
- package/dist/concurrency.js.map +1 -0
- package/dist/consumer-index.d.ts +6 -28
- package/dist/consumer-index.js +7 -8
- package/dist/consumer-index.js.map +1 -1
- package/dist/digest.d.ts +13 -0
- package/dist/digest.js +47 -0
- package/dist/digest.js.map +1 -0
- package/dist/fs-atomic.d.ts +7 -0
- package/dist/fs-atomic.js +30 -0
- package/dist/fs-atomic.js.map +1 -0
- package/dist/harness.js +2 -1
- package/dist/harness.js.map +1 -1
- package/dist/icon-mirror.d.ts +3 -2
- package/dist/icon-mirror.js +51 -63
- package/dist/icon-mirror.js.map +1 -1
- package/dist/icons.d.ts +2 -2
- package/dist/icons.js +4 -12
- package/dist/icons.js.map +1 -1
- package/dist/index.d.ts +16 -315
- package/dist/index.js +12 -1905
- package/dist/index.js.map +1 -1
- package/dist/integrity-manifest.d.ts +43 -13
- package/dist/integrity-manifest.js +28 -17
- package/dist/integrity-manifest.js.map +1 -1
- package/dist/json-values.d.ts +3 -0
- package/dist/json-values.js +10 -0
- package/dist/json-values.js.map +1 -0
- package/dist/plugin-release.d.ts +38 -0
- package/dist/plugin-release.js +321 -0
- package/dist/plugin-release.js.map +1 -0
- package/dist/plugin-skips.d.ts +19 -0
- package/dist/plugin-skips.js +65 -0
- package/dist/plugin-skips.js.map +1 -0
- package/dist/policy.d.ts +13 -0
- package/dist/policy.js +26 -0
- package/dist/policy.js.map +1 -0
- package/dist/portable-catalog.d.ts +4 -4
- package/dist/portable-catalog.js +14 -126
- package/dist/portable-catalog.js.map +1 -1
- package/dist/portable-verify.d.ts +2 -0
- package/dist/portable-verify.js +68 -0
- package/dist/portable-verify.js.map +1 -0
- package/dist/publication.d.ts +6 -3
- package/dist/publication.js +25 -12
- package/dist/publication.js.map +1 -1
- package/dist/published-output.d.ts +17 -0
- package/dist/published-output.js +49 -0
- package/dist/published-output.js.map +1 -0
- package/dist/site-icons.d.ts +2 -3
- package/dist/site-icons.js +4 -3
- package/dist/site-icons.js.map +1 -1
- package/dist/snapshots.d.ts +39 -0
- package/dist/snapshots.js +238 -0
- package/dist/snapshots.js.map +1 -0
- package/dist/source-resolution.d.ts +28 -0
- package/dist/source-resolution.js +158 -0
- package/dist/source-resolution.js.map +1 -0
- package/dist/sources.d.ts +5 -1
- package/dist/sources.js +21 -27
- package/dist/sources.js.map +1 -1
- package/dist/synchronize.d.ts +44 -0
- package/dist/synchronize.js +118 -0
- package/dist/synchronize.js.map +1 -0
- package/dist/taxonomy.d.ts +23 -0
- package/dist/taxonomy.js +62 -0
- package/dist/taxonomy.js.map +1 -0
- package/dist/upstream.d.ts +42 -0
- package/dist/upstream.js +179 -0
- package/dist/upstream.js.map +1 -0
- package/dist/verify-artifacts.d.ts +36 -0
- package/dist/verify-artifacts.js +286 -0
- package/dist/verify-artifacts.js.map +1 -0
- package/native/build-provenance.txt +5 -5
- package/native/staged-reader-win32-arm64.node +0 -0
- package/native/staged-reader-win32-x64.node +0 -0
- package/package.json +2 -4
- package/src/agent-plugins-catalog.test.ts +2 -2
- package/src/agent-plugins-catalog.ts +46 -49
- package/src/agent-plugins-schema.ts +11 -1
- package/src/artifacts.ts +234 -0
- package/src/build.ts +131 -0
- package/src/capabilities.ts +153 -0
- package/src/catalog-store.ts +1 -28
- package/src/concurrency.ts +19 -0
- package/src/consumer-index.test.ts +27 -21
- package/src/consumer-index.ts +12 -35
- package/src/digest.ts +55 -0
- package/src/fs-atomic.ts +32 -0
- package/src/harness.test.ts +4 -32
- package/src/harness.ts +2 -1
- package/src/icon-mirror.ts +57 -64
- package/src/icons.test.ts +0 -4
- package/src/icons.ts +4 -4
- package/src/index.test.ts +52 -48
- package/src/index.ts +50 -2583
- package/src/integrity-manifest.ts +41 -21
- package/src/json-values.ts +11 -0
- package/src/plugin-release.ts +457 -0
- package/src/plugin-skips.ts +110 -0
- package/src/policy.ts +39 -0
- package/src/portable-catalog.test.ts +0 -5
- package/src/portable-catalog.ts +16 -143
- package/src/portable-verify.ts +71 -0
- package/src/publication.test.ts +29 -1
- package/src/publication.ts +32 -13
- package/src/published-output.ts +60 -0
- package/src/site-icons.test.ts +0 -1
- package/src/site-icons.ts +4 -4
- package/src/snapshots.ts +287 -0
- package/src/source-digest.test.ts +3 -7
- package/src/source-resolution.ts +233 -0
- package/src/sources-json.test.ts +0 -1
- package/src/sources.ts +27 -33
- package/src/synchronize.ts +213 -0
- package/src/taxonomy.ts +87 -0
- package/src/upstream.ts +194 -0
- package/src/verify-artifacts.ts +365 -0
- package/src/write-artifacts.test.ts +0 -1
- package/dist/source-digest.d.ts +0 -2
- package/dist/source-digest.js +0 -12
- package/dist/source-digest.js.map +0 -1
- package/src/source-digest.ts +0 -14
package/src/snapshots.ts
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
import { promises as fs } from 'node:fs'
|
|
2
|
+
import { join, relative, resolve } from 'node:path'
|
|
3
|
+
import { Readable } from 'node:stream'
|
|
4
|
+
import { createGunzip } from 'node:zlib'
|
|
5
|
+
import { extract } from 'tar-stream'
|
|
6
|
+
import type { CatalogPolicy } from './policy.js'
|
|
7
|
+
import { PluginSafetyError } from './plugin-skips.js'
|
|
8
|
+
import { canonicalRepositoryUrl, safeRelativePath, type SourceConfig } from './sources.js'
|
|
9
|
+
import { fetchUpstream, GITHUB_REPOSITORY_HOSTS, upstreamRequestHeaders } from './upstream.js'
|
|
10
|
+
|
|
11
|
+
export interface Snapshot {
|
|
12
|
+
readonly files: ReadonlyMap<string, Uint8Array>
|
|
13
|
+
readonly symlinks: readonly string[]
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface SnapshotLoader {
|
|
17
|
+
load(repositoryUrl: string, commitSha: string, pluginSubdirectory: string): Promise<Snapshot>
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Ceiling on bytes read out of one repository archive.
|
|
22
|
+
*
|
|
23
|
+
* The per-plugin policy still bounds what any single plugin may contain; this
|
|
24
|
+
* only stops a decompression bomb from exhausting the build host, and it is
|
|
25
|
+
* enforced while the stream is read rather than after it is buffered.
|
|
26
|
+
*/
|
|
27
|
+
const MAX_ARCHIVE_BYTES = 512 * 1024 * 1024
|
|
28
|
+
|
|
29
|
+
function concatChunks(chunks: readonly Uint8Array[], length: number): Uint8Array {
|
|
30
|
+
const out = new Uint8Array(length)
|
|
31
|
+
let offset = 0
|
|
32
|
+
for (const chunk of chunks) {
|
|
33
|
+
out.set(chunk, offset)
|
|
34
|
+
offset += chunk.byteLength
|
|
35
|
+
}
|
|
36
|
+
return out
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Read one GitHub repository tarball into path -> bytes.
|
|
41
|
+
*
|
|
42
|
+
* GitHub wraps every entry in a single top-level `<repo>-<sha>/` directory,
|
|
43
|
+
* which is stripped so the keys match the tree API's repository-relative
|
|
44
|
+
* paths. Only regular files are taken: directories, symlinks, hardlinks and
|
|
45
|
+
* the `pax_global_header` record are skipped, so a link can never be
|
|
46
|
+
* materialized as content. Every remaining path goes through
|
|
47
|
+
* `safeRelativePath`, so an archive that tries to escape the repository is
|
|
48
|
+
* rejected rather than partially trusted. This is a whole-repository archive
|
|
49
|
+
* shared across plugins, so an oversized unrelated file is drained and
|
|
50
|
+
* counted against the archive budget but not retained. The selected plugin's
|
|
51
|
+
* files are checked against the tree before the archive is fetched and again
|
|
52
|
+
* when its snapshot is validated.
|
|
53
|
+
*/
|
|
54
|
+
export async function readArchiveFiles(
|
|
55
|
+
archive: Uint8Array,
|
|
56
|
+
policy: CatalogPolicy
|
|
57
|
+
): Promise<Map<string, Uint8Array>> {
|
|
58
|
+
const files = new Map<string, Uint8Array>()
|
|
59
|
+
let totalBytes = 0
|
|
60
|
+
const parser = extract()
|
|
61
|
+
const gunzip = createGunzip()
|
|
62
|
+
const settled = new Promise<void>((resolveDone, rejectDone) => {
|
|
63
|
+
const fail = (error: unknown): void => {
|
|
64
|
+
gunzip.destroy()
|
|
65
|
+
parser.destroy()
|
|
66
|
+
rejectDone(error)
|
|
67
|
+
}
|
|
68
|
+
parser.on('entry', (header, stream, next) => {
|
|
69
|
+
const skip = (): void => {
|
|
70
|
+
stream.on('end', next)
|
|
71
|
+
stream.resume()
|
|
72
|
+
}
|
|
73
|
+
if (header.type !== 'file') return skip()
|
|
74
|
+
const segments = header.name.split('/')
|
|
75
|
+
if (segments.length < 2) return skip()
|
|
76
|
+
let path: string
|
|
77
|
+
try {
|
|
78
|
+
path = safeRelativePath(segments.slice(1).join('/'), 'TREE_PATH_INVALID')
|
|
79
|
+
} catch (error) {
|
|
80
|
+
return fail(error)
|
|
81
|
+
}
|
|
82
|
+
const chunks: Uint8Array[] = []
|
|
83
|
+
let length = 0
|
|
84
|
+
stream.on('data', (chunk: unknown) => {
|
|
85
|
+
const bytes = chunk as Uint8Array
|
|
86
|
+
length += bytes.byteLength
|
|
87
|
+
if (totalBytes + length > MAX_ARCHIVE_BYTES) {
|
|
88
|
+
fail(new Error('PLUGIN_SIZE_POLICY: archive'))
|
|
89
|
+
return
|
|
90
|
+
}
|
|
91
|
+
if (length <= policy.maxFileBytes) chunks.push(bytes)
|
|
92
|
+
else chunks.length = 0
|
|
93
|
+
})
|
|
94
|
+
stream.on('error', fail)
|
|
95
|
+
stream.on('end', () => {
|
|
96
|
+
try {
|
|
97
|
+
totalBytes += length
|
|
98
|
+
if (length <= policy.maxFileBytes) files.set(path, concatChunks(chunks, length))
|
|
99
|
+
next()
|
|
100
|
+
} catch (error) {
|
|
101
|
+
fail(error)
|
|
102
|
+
}
|
|
103
|
+
})
|
|
104
|
+
stream.resume()
|
|
105
|
+
})
|
|
106
|
+
parser.on('error', fail)
|
|
107
|
+
gunzip.on('error', fail)
|
|
108
|
+
parser.on('finish', () => resolveDone())
|
|
109
|
+
})
|
|
110
|
+
Readable.from([archive]).pipe(gunzip).pipe(parser)
|
|
111
|
+
await settled
|
|
112
|
+
return files
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export class NetworkSnapshotLoader implements SnapshotLoader {
|
|
116
|
+
readonly #policy: CatalogPolicy
|
|
117
|
+
readonly #trees = new Map<string, Promise<readonly TreeEntry[]>>()
|
|
118
|
+
readonly #archives = new Map<string, Promise<ReadonlyMap<string, Uint8Array>>>()
|
|
119
|
+
|
|
120
|
+
constructor(policy: CatalogPolicy) {
|
|
121
|
+
this.#policy = policy
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
async load(
|
|
125
|
+
repositoryUrl: string,
|
|
126
|
+
commitSha: string,
|
|
127
|
+
pluginSubdirectory: string
|
|
128
|
+
): Promise<Snapshot> {
|
|
129
|
+
const parsed = new URL(canonicalRepositoryUrl(repositoryUrl))
|
|
130
|
+
if (!GITHUB_REPOSITORY_HOSTS.includes(parsed.hostname))
|
|
131
|
+
throw new Error(`GIT_HOST_UNSUPPORTED: ${parsed.hostname}`)
|
|
132
|
+
const [owner, repo] = parsed.pathname.split('/').filter(Boolean)
|
|
133
|
+
if (!owner || !repo) throw new Error('GIT_REPOSITORY_INVALID')
|
|
134
|
+
const key = `${owner}/${repo}@${commitSha}`
|
|
135
|
+
let pendingTree = this.#trees.get(key)
|
|
136
|
+
if (!pendingTree) {
|
|
137
|
+
pendingTree = (async () => {
|
|
138
|
+
const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${commitSha}?recursive=1`
|
|
139
|
+
const response = await fetchUpstream(treeUrl, {
|
|
140
|
+
headers: upstreamRequestHeaders(treeUrl),
|
|
141
|
+
})
|
|
142
|
+
if (!response.ok)
|
|
143
|
+
throw new PluginSafetyError('PLUGIN_SOURCE_UNAVAILABLE', [`${owner}/${repo}`])
|
|
144
|
+
const payload = (await response.json()) as { truncated?: boolean; tree?: TreeEntry[] }
|
|
145
|
+
if (payload.truncated) throw new Error(`GIT_TREE_TRUNCATED: ${owner}/${repo}@${commitSha}`)
|
|
146
|
+
return (payload.tree ?? []).map((entry) => ({
|
|
147
|
+
...entry,
|
|
148
|
+
path: safeRelativePath(entry.path, 'TREE_PATH_INVALID'),
|
|
149
|
+
}))
|
|
150
|
+
})().catch((error: unknown) => {
|
|
151
|
+
this.#trees.delete(key)
|
|
152
|
+
throw error
|
|
153
|
+
})
|
|
154
|
+
this.#trees.set(key, pendingTree)
|
|
155
|
+
}
|
|
156
|
+
const tree = await pendingTree
|
|
157
|
+
const prefix = pluginSubdirectory ? `${safeRelativePath(pluginSubdirectory)}/` : ''
|
|
158
|
+
const submodules = tree.filter(
|
|
159
|
+
(entry) =>
|
|
160
|
+
entry.path.startsWith(prefix) && (entry.type === 'commit' || entry.mode === '160000')
|
|
161
|
+
)
|
|
162
|
+
if (submodules.length > 0)
|
|
163
|
+
throw new Error(`GIT_SUBMODULE_UNSUPPORTED: ${repositoryUrl}:${pluginSubdirectory}`)
|
|
164
|
+
const selected = tree.filter((entry) => entry.type === 'blob' && entry.path.startsWith(prefix))
|
|
165
|
+
if (selected.length === 0)
|
|
166
|
+
throw new Error(`PLUGIN_PATH_NOT_FOUND: ${repositoryUrl}:${pluginSubdirectory}`)
|
|
167
|
+
const files = new Map<string, Uint8Array>()
|
|
168
|
+
const symlinks = tree
|
|
169
|
+
.filter((entry) => entry.mode === '120000' && entry.path.startsWith(prefix))
|
|
170
|
+
.map((entry) => entry.path)
|
|
171
|
+
if (selected.length > this.#policy.maxFilesPerPlugin) throw new Error('PLUGIN_TOO_MANY_FILES')
|
|
172
|
+
let totalSize = 0
|
|
173
|
+
for (const entry of selected) {
|
|
174
|
+
const size = entry.size ?? 0
|
|
175
|
+
if (size > this.#policy.maxFileBytes || totalSize + size > this.#policy.maxBytesPerPlugin)
|
|
176
|
+
throw new Error(`PLUGIN_SIZE_POLICY: ${entry.path}`)
|
|
177
|
+
totalSize += size
|
|
178
|
+
}
|
|
179
|
+
// The tree above already bounded this plugin before a byte was pulled, so
|
|
180
|
+
// the archive is the only unbounded read left and it is capped while it
|
|
181
|
+
// streams. One archive per repository replaces one request per file.
|
|
182
|
+
const archive = await this.#archiveFiles(key, owner, repo, commitSha)
|
|
183
|
+
for (const entry of selected) {
|
|
184
|
+
const bytes = archive.get(entry.path)
|
|
185
|
+
if (!bytes) throw new PluginSafetyError('PLUGIN_FILE_UNAVAILABLE', [entry.path])
|
|
186
|
+
if (bytes.byteLength > this.#policy.maxFileBytes)
|
|
187
|
+
throw new Error(`PLUGIN_FILE_TOO_LARGE: ${entry.path}`)
|
|
188
|
+
files.set(
|
|
189
|
+
prefix ? relative(prefix.slice(0, -1), entry.path).replaceAll('\\', '/') : entry.path,
|
|
190
|
+
bytes
|
|
191
|
+
)
|
|
192
|
+
}
|
|
193
|
+
return { files, symlinks }
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
async #archiveFiles(
|
|
197
|
+
key: string,
|
|
198
|
+
owner: string,
|
|
199
|
+
repo: string,
|
|
200
|
+
commitSha: string
|
|
201
|
+
): Promise<ReadonlyMap<string, Uint8Array>> {
|
|
202
|
+
const cached = this.#archives.get(key)
|
|
203
|
+
if (cached) return cached
|
|
204
|
+
const pending = (async () => {
|
|
205
|
+
const url = `https://api.github.com/repos/${owner}/${repo}/tarball/${commitSha}`
|
|
206
|
+
const response = await fetchUpstream(url, { headers: upstreamRequestHeaders(url) })
|
|
207
|
+
if (!response.ok)
|
|
208
|
+
throw new PluginSafetyError('PLUGIN_SOURCE_UNAVAILABLE', [`${owner}/${repo}`])
|
|
209
|
+
return readArchiveFiles(new Uint8Array(await response.arrayBuffer()), this.#policy)
|
|
210
|
+
})().catch((error: unknown) => {
|
|
211
|
+
this.#archives.delete(key)
|
|
212
|
+
throw error
|
|
213
|
+
})
|
|
214
|
+
this.#archives.set(key, pending)
|
|
215
|
+
return pending
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
interface TreeEntry {
|
|
220
|
+
readonly path: string
|
|
221
|
+
readonly mode?: string
|
|
222
|
+
readonly type?: string
|
|
223
|
+
readonly size?: number
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
export class FixtureSnapshotLoader implements SnapshotLoader {
|
|
227
|
+
readonly #root: string
|
|
228
|
+
readonly #sources: ReadonlyMap<string, SourceConfig>
|
|
229
|
+
|
|
230
|
+
constructor(root: string, sources: readonly SourceConfig[]) {
|
|
231
|
+
this.#root = resolve(root)
|
|
232
|
+
this.#sources = new Map(
|
|
233
|
+
sources.map((source) => [canonicalRepositoryUrl(source.repositoryUrl), source])
|
|
234
|
+
)
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
async load(
|
|
238
|
+
repositoryUrl: string,
|
|
239
|
+
_commitSha: string,
|
|
240
|
+
pluginSubdirectory: string
|
|
241
|
+
): Promise<Snapshot> {
|
|
242
|
+
const source = this.#sources.get(canonicalRepositoryUrl(repositoryUrl))
|
|
243
|
+
if (!source?.fixturePath) throw new Error(`FIXTURE_REPOSITORY_NOT_FOUND: ${repositoryUrl}`)
|
|
244
|
+
return snapshotFromDirectory(join(this.#root, source.fixturePath), pluginSubdirectory)
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export class DirectorySnapshotLoader implements SnapshotLoader {
|
|
249
|
+
async load(
|
|
250
|
+
_repositoryUrl: string,
|
|
251
|
+
_commitSha: string,
|
|
252
|
+
pluginSubdirectory: string
|
|
253
|
+
): Promise<Snapshot> {
|
|
254
|
+
throw new Error(`DIRECTORY_LOADER_REQUIRES_ROOT: ${pluginSubdirectory}`)
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
export async function snapshotFromDirectory(
|
|
259
|
+
root: string,
|
|
260
|
+
pluginSubdirectory: string
|
|
261
|
+
): Promise<Snapshot> {
|
|
262
|
+
const directory = resolve(root, pluginSubdirectory)
|
|
263
|
+
const files = new Map<string, Uint8Array>()
|
|
264
|
+
const symlinks: string[] = []
|
|
265
|
+
const rootStat = await fs.lstat(directory)
|
|
266
|
+
if (rootStat.isSymbolicLink()) return { files, symlinks: [pluginSubdirectory || '.'] }
|
|
267
|
+
await walk(directory, directory, files, symlinks)
|
|
268
|
+
return { files, symlinks }
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
async function walk(
|
|
272
|
+
root: string,
|
|
273
|
+
current: string,
|
|
274
|
+
files: Map<string, Uint8Array>,
|
|
275
|
+
symlinks: string[]
|
|
276
|
+
): Promise<void> {
|
|
277
|
+
for (const entry of await fs.readdir(current, { withFileTypes: true })) {
|
|
278
|
+
const absolute = join(current, entry.name)
|
|
279
|
+
const relativePath = safeRelativePath(relative(root, absolute), 'DIRECTORY_PATH_INVALID')
|
|
280
|
+
if (entry.isSymbolicLink()) {
|
|
281
|
+
symlinks.push(relativePath)
|
|
282
|
+
continue
|
|
283
|
+
}
|
|
284
|
+
if (entry.isDirectory()) await walk(root, absolute, files, symlinks)
|
|
285
|
+
else if (entry.isFile()) files.set(relativePath, await fs.readFile(absolute))
|
|
286
|
+
}
|
|
287
|
+
}
|
|
@@ -11,20 +11,17 @@ test('source digests agree across consumer locales and equivalent Unicode path o
|
|
|
11
11
|
try {
|
|
12
12
|
const entry = join(directory, 'entry.ts')
|
|
13
13
|
const output = join(directory, 'consumer.mjs')
|
|
14
|
-
await writeFile(
|
|
15
|
-
entry,
|
|
16
|
-
`export { bytesDigest } from '../index.ts'\nexport { sourceDigest } from '../agent-plugins-catalog.ts'\n`
|
|
17
|
-
)
|
|
14
|
+
await writeFile(entry, `export { sourceDigest } from '../digest.ts'\n`)
|
|
18
15
|
const build = await Bun.build({ entrypoints: [entry], target: 'node', packages: 'external' })
|
|
19
16
|
expect(build.success).toBe(true)
|
|
20
17
|
await Bun.write(output, build.outputs[0]!)
|
|
21
18
|
const script = `
|
|
22
|
-
import {
|
|
19
|
+
import { sourceDigest } from ${JSON.stringify(pathToFileURL(output).href)};
|
|
23
20
|
const paths = ['z.txt', 'ä.txt', 'é.txt', 'e\\u0301.txt'];
|
|
24
21
|
const entries = paths.map((path) => [path, new TextEncoder().encode(path)]);
|
|
25
22
|
const forward = new Map(entries), reverse = new Map([...entries].reverse());
|
|
26
23
|
console.log(JSON.stringify([
|
|
27
|
-
|
|
24
|
+
sourceDigest(forward), sourceDigest(reverse)
|
|
28
25
|
]));
|
|
29
26
|
`
|
|
30
27
|
const results = ['en_US.UTF-8', 'sv_SE.UTF-8'].map((locale) => {
|
|
@@ -34,7 +31,6 @@ test('source digests agree across consumer locales and equivalent Unicode path o
|
|
|
34
31
|
})
|
|
35
32
|
if (result.error) throw result.error
|
|
36
33
|
if (result.status !== 0) throw new Error(result.stderr)
|
|
37
|
-
expect(result.status).toBe(0)
|
|
38
34
|
return JSON.parse(result.stdout) as string[]
|
|
39
35
|
})
|
|
40
36
|
expect(new Set(results.flat()).size).toBe(1)
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import { promises as fs } from 'node:fs'
|
|
2
|
+
import { join, resolve } from 'node:path'
|
|
3
|
+
import { mapWithConcurrency } from './concurrency.js'
|
|
4
|
+
import { contentId, digest, ordinalCompare } from './digest.js'
|
|
5
|
+
import { SourcesLockSchema, type SourceLockEntry, type SourcesLock } from './schema.js'
|
|
6
|
+
import {
|
|
7
|
+
canonicalRepositoryUrl,
|
|
8
|
+
parseJsonDocument,
|
|
9
|
+
parseMarketplaceManifest,
|
|
10
|
+
safeRelativePath,
|
|
11
|
+
type ParsedMarketplace,
|
|
12
|
+
type PluginSourceSpec,
|
|
13
|
+
type SourceConfig,
|
|
14
|
+
} from './sources.js'
|
|
15
|
+
import { externalGitRef, fetchText, rawContentUrl, resolveGitRef } from './upstream.js'
|
|
16
|
+
|
|
17
|
+
export interface ResolvedSource {
|
|
18
|
+
readonly config: SourceConfig
|
|
19
|
+
readonly commitSha: string
|
|
20
|
+
readonly manifestText: string
|
|
21
|
+
readonly parsed: ParsedMarketplace
|
|
22
|
+
readonly pluginPins: readonly PluginSourcePin[]
|
|
23
|
+
readonly retrievedAt: string
|
|
24
|
+
readonly manifestDigest: string
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface PluginSourcePin {
|
|
28
|
+
readonly pluginName: string
|
|
29
|
+
readonly repositoryUrl: string
|
|
30
|
+
readonly pluginSubdirectory: string
|
|
31
|
+
readonly resolvedCommitSha: string
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export async function resolvePluginSource(
|
|
35
|
+
pluginName: string,
|
|
36
|
+
spec: PluginSourceSpec,
|
|
37
|
+
source: ResolvedSource,
|
|
38
|
+
resolveExternalRefs: boolean
|
|
39
|
+
): Promise<ResolvedPluginSource> {
|
|
40
|
+
if (spec.kind === 'local')
|
|
41
|
+
return {
|
|
42
|
+
repositoryUrl: canonicalRepositoryUrl(source.config.repositoryUrl),
|
|
43
|
+
subdirectory: spec.path,
|
|
44
|
+
commitSha: source.commitSha,
|
|
45
|
+
}
|
|
46
|
+
const canonicalUrl = canonicalRepositoryUrl(spec.repositoryUrl)
|
|
47
|
+
const pin = source.pluginPins.find(
|
|
48
|
+
(candidate) =>
|
|
49
|
+
candidate.pluginName === pluginName &&
|
|
50
|
+
candidate.repositoryUrl === canonicalUrl &&
|
|
51
|
+
candidate.pluginSubdirectory === spec.subdirectory
|
|
52
|
+
)
|
|
53
|
+
const commitSha =
|
|
54
|
+
pin?.resolvedCommitSha ??
|
|
55
|
+
spec.sha ??
|
|
56
|
+
(resolveExternalRefs
|
|
57
|
+
? await resolveGitRef(spec.repositoryUrl, externalGitRef(spec.ref))
|
|
58
|
+
: source.commitSha)
|
|
59
|
+
return {
|
|
60
|
+
repositoryUrl: canonicalUrl,
|
|
61
|
+
subdirectory: pin?.pluginSubdirectory ?? spec.subdirectory,
|
|
62
|
+
commitSha,
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export interface ResolvedPluginSource {
|
|
67
|
+
readonly repositoryUrl: string
|
|
68
|
+
readonly subdirectory: string
|
|
69
|
+
readonly commitSha: string
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** The fields that differ between two lock entries for the same source. */
|
|
73
|
+
export function lockChangedFields(previous: SourceLockEntry, current: SourceLockEntry): string[] {
|
|
74
|
+
const fields = [
|
|
75
|
+
'repositoryUrl',
|
|
76
|
+
'defaultBranch',
|
|
77
|
+
'manifestPath',
|
|
78
|
+
'marketplaceDialect',
|
|
79
|
+
'resolvedCommitSha',
|
|
80
|
+
'sourceManifestDigest',
|
|
81
|
+
'synchronizationStatus',
|
|
82
|
+
] as const
|
|
83
|
+
const changed: string[] = fields.filter((field) => previous[field] !== current[field])
|
|
84
|
+
if (digest(sortedPins(previous)) !== digest(sortedPins(current))) changed.push('pluginPins')
|
|
85
|
+
return changed
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function sourceLockChanged(previous: SourceLockEntry, current: SourceLockEntry): boolean {
|
|
89
|
+
return lockChangedFields(previous, current).length > 0
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function sortedPins(source: SourceLockEntry): string[] {
|
|
93
|
+
return source.pluginPins
|
|
94
|
+
.map((pin) =>
|
|
95
|
+
JSON.stringify([
|
|
96
|
+
pin.pluginName,
|
|
97
|
+
pin.repositoryUrl,
|
|
98
|
+
pin.pluginSubdirectory,
|
|
99
|
+
pin.resolvedCommitSha,
|
|
100
|
+
])
|
|
101
|
+
)
|
|
102
|
+
.toSorted()
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export function createSourcesLock(sources: readonly ResolvedSource[]): SourcesLock {
|
|
106
|
+
const entries: SourceLockEntry[] = sources
|
|
107
|
+
.map((source) => ({
|
|
108
|
+
sourceId: source.config.sourceId,
|
|
109
|
+
repositoryUrl: canonicalRepositoryUrl(source.config.repositoryUrl),
|
|
110
|
+
defaultBranch: source.config.defaultBranch,
|
|
111
|
+
resolvedCommitSha: source.commitSha,
|
|
112
|
+
manifestPath: source.config.manifestPath,
|
|
113
|
+
marketplaceDialect: source.config.marketplaceDialect,
|
|
114
|
+
sourceManifestDigest: source.manifestDigest,
|
|
115
|
+
retrievedAt: source.retrievedAt,
|
|
116
|
+
synchronizationStatus: 'synchronized' as const,
|
|
117
|
+
pluginPins: source.pluginPins.map((pin) => ({ ...pin })),
|
|
118
|
+
}))
|
|
119
|
+
.toSorted((left, right) => left.sourceId.localeCompare(right.sourceId))
|
|
120
|
+
const body = { schemaVersion: 1 as const, sources: entries }
|
|
121
|
+
return SourcesLockSchema.parse({
|
|
122
|
+
...body,
|
|
123
|
+
lockId: contentId('lock', body),
|
|
124
|
+
})
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export async function resolveSource(
|
|
128
|
+
config: SourceConfig,
|
|
129
|
+
mode: 'live' | 'offline',
|
|
130
|
+
fixtureRoot: string | undefined,
|
|
131
|
+
lock: SourcesLock | undefined,
|
|
132
|
+
fromLock: boolean
|
|
133
|
+
): Promise<ResolvedSource> {
|
|
134
|
+
const manifestPath = safeRelativePath(config.manifestPath, 'SOURCE_MANIFEST_PATH_INVALID')
|
|
135
|
+
const locked = lock?.sources.find((entry) => entry.sourceId === config.sourceId)
|
|
136
|
+
if (fromLock && !locked) throw new Error(`SOURCE_LOCK_ENTRY_MISSING: ${config.sourceId}`)
|
|
137
|
+
const commitSha =
|
|
138
|
+
fromLock && locked
|
|
139
|
+
? locked.resolvedCommitSha
|
|
140
|
+
: mode === 'offline'
|
|
141
|
+
? (config.fixtureCommitSha ?? locked?.resolvedCommitSha)
|
|
142
|
+
: await resolveGitRef(config.repositoryUrl, `refs/heads/${config.defaultBranch}`)
|
|
143
|
+
if (!commitSha || !/^[a-f0-9]{40}$/.test(commitSha))
|
|
144
|
+
throw new Error(`SOURCE_COMMIT_UNRESOLVED: ${config.sourceId}`)
|
|
145
|
+
const manifestText =
|
|
146
|
+
mode === 'offline'
|
|
147
|
+
? await readFixtureManifest({ ...config, manifestPath }, fixtureRoot)
|
|
148
|
+
: await fetchText(rawContentUrl(config.repositoryUrl, commitSha, manifestPath))
|
|
149
|
+
const parsed = parseMarketplaceManifest(parseJsonDocument(manifestText), config)
|
|
150
|
+
if (fromLock && locked && digest(manifestText) !== locked.sourceManifestDigest)
|
|
151
|
+
throw new Error(`SOURCE_LOCK_MANIFEST_MISMATCH: ${config.sourceId}`)
|
|
152
|
+
const pluginPins = await resolvePluginPins({
|
|
153
|
+
config,
|
|
154
|
+
parsed,
|
|
155
|
+
sourceCommitSha: commitSha,
|
|
156
|
+
lockedPins: locked?.pluginPins,
|
|
157
|
+
fromLock,
|
|
158
|
+
resolveExternalRefs: mode === 'live',
|
|
159
|
+
})
|
|
160
|
+
return {
|
|
161
|
+
config,
|
|
162
|
+
commitSha,
|
|
163
|
+
manifestText,
|
|
164
|
+
parsed,
|
|
165
|
+
pluginPins,
|
|
166
|
+
retrievedAt:
|
|
167
|
+
fromLock && locked
|
|
168
|
+
? locked.retrievedAt
|
|
169
|
+
: mode === 'offline'
|
|
170
|
+
? '2026-01-01T00:00:00.000Z'
|
|
171
|
+
: new Date().toISOString(),
|
|
172
|
+
manifestDigest: digest(manifestText),
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
async function resolvePluginPins(input: {
|
|
177
|
+
readonly config: SourceConfig
|
|
178
|
+
readonly parsed: ParsedMarketplace
|
|
179
|
+
readonly sourceCommitSha: string
|
|
180
|
+
readonly lockedPins: readonly PluginSourcePin[] | undefined
|
|
181
|
+
readonly fromLock: boolean
|
|
182
|
+
readonly resolveExternalRefs: boolean
|
|
183
|
+
}): Promise<PluginSourcePin[]> {
|
|
184
|
+
const pins = await mapWithConcurrency(
|
|
185
|
+
input.parsed.entries,
|
|
186
|
+
8,
|
|
187
|
+
async (entry): Promise<PluginSourcePin> => {
|
|
188
|
+
if (entry.source.kind === 'local') {
|
|
189
|
+
return {
|
|
190
|
+
pluginName: entry.name,
|
|
191
|
+
repositoryUrl: canonicalRepositoryUrl(input.config.repositoryUrl),
|
|
192
|
+
pluginSubdirectory: entry.source.path,
|
|
193
|
+
resolvedCommitSha: input.sourceCommitSha,
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const repositoryUrl = canonicalRepositoryUrl(entry.source.repositoryUrl)
|
|
198
|
+
const pluginSubdirectory = entry.source.subdirectory
|
|
199
|
+
// Existing locks describe the prior sync; only explicit replay freezes revisions.
|
|
200
|
+
const locked = input.fromLock
|
|
201
|
+
? input.lockedPins?.find((pin) => pin.pluginName === entry.name)
|
|
202
|
+
: undefined
|
|
203
|
+
if (input.fromLock && !locked)
|
|
204
|
+
throw new Error(`SOURCE_LOCK_PLUGIN_PIN_MISSING: ${input.config.sourceId}:${entry.name}`)
|
|
205
|
+
if (locked) {
|
|
206
|
+
if (
|
|
207
|
+
locked.repositoryUrl !== repositoryUrl ||
|
|
208
|
+
locked.pluginSubdirectory !== pluginSubdirectory
|
|
209
|
+
) {
|
|
210
|
+
throw new Error(`SOURCE_LOCK_PLUGIN_PIN_MISMATCH: ${input.config.sourceId}:${entry.name}`)
|
|
211
|
+
}
|
|
212
|
+
return locked
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
const resolvedCommitSha =
|
|
216
|
+
entry.source.sha ??
|
|
217
|
+
(input.resolveExternalRefs
|
|
218
|
+
? await resolveGitRef(repositoryUrl, externalGitRef(entry.source.ref))
|
|
219
|
+
: input.sourceCommitSha)
|
|
220
|
+
return { pluginName: entry.name, repositoryUrl, pluginSubdirectory, resolvedCommitSha }
|
|
221
|
+
}
|
|
222
|
+
)
|
|
223
|
+
return pins.toSorted((left, right) => ordinalCompare(left.pluginName, right.pluginName))
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
async function readFixtureManifest(
|
|
227
|
+
config: SourceConfig,
|
|
228
|
+
fixtureRoot: string | undefined
|
|
229
|
+
): Promise<string> {
|
|
230
|
+
if (!fixtureRoot || !config.fixturePath)
|
|
231
|
+
throw new Error(`FIXTURE_ROOT_REQUIRED: ${config.sourceId}`)
|
|
232
|
+
return fs.readFile(join(resolve(fixtureRoot), config.fixturePath, config.manifestPath), 'utf8')
|
|
233
|
+
}
|
package/src/sources-json.test.ts
CHANGED
package/src/sources.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { digest, ordinalCompare } from './digest.js'
|
|
2
|
+
import { isString, stringValue } from './json-values.js'
|
|
3
|
+
import type { MarketplaceDialectSchema } from './schema.js'
|
|
4
|
+
import type { z } from 'zod'
|
|
2
5
|
|
|
3
|
-
export type MarketplaceDialect =
|
|
6
|
+
export type MarketplaceDialect = z.output<typeof MarketplaceDialectSchema>
|
|
4
7
|
|
|
5
8
|
export interface SourceConfig {
|
|
6
9
|
readonly sourceId: string
|
|
@@ -44,6 +47,8 @@ export interface ParsedMarketplace {
|
|
|
44
47
|
readonly marketplaceDescription: string
|
|
45
48
|
readonly owner: readonly string[]
|
|
46
49
|
readonly entries: readonly MarketplacePluginEntry[]
|
|
50
|
+
/** Entries left out because their source kind cannot be installed from a repository. */
|
|
51
|
+
readonly unsupportedEntries?: readonly string[]
|
|
47
52
|
readonly raw: Record<string, unknown>
|
|
48
53
|
}
|
|
49
54
|
|
|
@@ -62,7 +67,23 @@ export function parseMarketplaceManifest(input: unknown, source: SourceConfig):
|
|
|
62
67
|
const plugins = root.plugins
|
|
63
68
|
if (!Array.isArray(plugins)) throw new SourceAdapterError('MARKETPLACE_PLUGINS_REQUIRED')
|
|
64
69
|
|
|
65
|
-
|
|
70
|
+
// Upstream marketplaces can list plugins that install some other way, such as by running a
|
|
71
|
+
// command. Those stay out of the catalog, and they must not stop the rest of it from building.
|
|
72
|
+
const entries: MarketplacePluginEntry[] = []
|
|
73
|
+
const unsupportedEntries: string[] = []
|
|
74
|
+
for (const [index, value] of plugins.entries()) {
|
|
75
|
+
try {
|
|
76
|
+
entries.push(parsePluginEntry(value, source, index))
|
|
77
|
+
} catch (error) {
|
|
78
|
+
if (!(error instanceof SourceAdapterError) || error.code !== 'PLUGIN_SOURCE_KIND_UNSUPPORTED')
|
|
79
|
+
throw error
|
|
80
|
+
unsupportedEntries.push(error.details.join(': '))
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
if (unsupportedEntries.length > 0)
|
|
84
|
+
console.error(
|
|
85
|
+
`${source.sourceId}: left out plugins with unsupported source kinds: ${unsupportedEntries.join(', ')}`
|
|
86
|
+
)
|
|
66
87
|
const duplicateNames = new Set<string>()
|
|
67
88
|
for (const entry of entries) {
|
|
68
89
|
if (duplicateNames.has(entry.name))
|
|
@@ -72,8 +93,9 @@ export function parseMarketplaceManifest(input: unknown, source: SourceConfig):
|
|
|
72
93
|
return {
|
|
73
94
|
marketplaceName: stringValue(root.name) ?? source.displayName,
|
|
74
95
|
marketplaceDescription: stringValue(root.description) ?? '',
|
|
75
|
-
owner:
|
|
96
|
+
owner: authorNames(root.owner),
|
|
76
97
|
entries,
|
|
98
|
+
...(unsupportedEntries.length > 0 ? { unsupportedEntries } : {}),
|
|
77
99
|
raw: root,
|
|
78
100
|
}
|
|
79
101
|
}
|
|
@@ -302,18 +324,8 @@ function requiredString(input: unknown, code: string, detail: string): string {
|
|
|
302
324
|
return result
|
|
303
325
|
}
|
|
304
326
|
|
|
305
|
-
function stringValue(input: unknown): string | undefined {
|
|
306
|
-
return typeof input === 'string' && input.trim() ? input.trim() : undefined
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
function isString(input: unknown): input is string {
|
|
310
|
-
return typeof input === 'string' && input.trim().length > 0
|
|
311
|
-
}
|
|
312
|
-
|
|
313
327
|
function uniqueStrings(input: readonly string[]): string[] {
|
|
314
|
-
return [...new Set(input.map((value) => value.trim()).filter(Boolean))].toSorted(
|
|
315
|
-
a < b ? -1 : a > b ? 1 : 0
|
|
316
|
-
)
|
|
328
|
+
return [...new Set(input.map((value) => value.trim()).filter(Boolean))].toSorted(ordinalCompare)
|
|
317
329
|
}
|
|
318
330
|
|
|
319
331
|
function authorNames(input: unknown): string[] {
|
|
@@ -327,10 +339,6 @@ function authorNames(input: unknown): string[] {
|
|
|
327
339
|
return []
|
|
328
340
|
}
|
|
329
341
|
|
|
330
|
-
function ownerNames(input: unknown): string[] {
|
|
331
|
-
return authorNames(input)
|
|
332
|
-
}
|
|
333
|
-
|
|
334
342
|
function safeOptionalUrl(input: unknown): string | undefined {
|
|
335
343
|
if (!isString(input)) return undefined
|
|
336
344
|
try {
|
|
@@ -344,17 +352,3 @@ function safeOptionalUrl(input: unknown): string | undefined {
|
|
|
344
352
|
function looksLikeUrl(input: string): boolean {
|
|
345
353
|
return /^https?:\/\//i.test(input)
|
|
346
354
|
}
|
|
347
|
-
|
|
348
|
-
function digest(value: unknown): string {
|
|
349
|
-
return `sha256:${createHash('sha256').update(canonicalJson(value)).digest('hex')}`
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
function canonicalJson(value: unknown): string {
|
|
353
|
-
if (value === null || typeof value !== 'object') return JSON.stringify(value)
|
|
354
|
-
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`
|
|
355
|
-
const object = value as Record<string, unknown>
|
|
356
|
-
return `{${Object.keys(object)
|
|
357
|
-
.toSorted()
|
|
358
|
-
.map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`)
|
|
359
|
-
.join(',')}}`
|
|
360
|
-
}
|