@adea-ai/plugins 4.0.0 → 4.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/dist/agent-plugins-catalog.d.ts +2 -4
  2. package/dist/agent-plugins-catalog.js +42 -50
  3. package/dist/agent-plugins-catalog.js.map +1 -1
  4. package/dist/agent-plugins-schema.d.ts +8 -0
  5. package/dist/agent-plugins-schema.js +10 -1
  6. package/dist/agent-plugins-schema.js.map +1 -1
  7. package/dist/artifacts.d.ts +57 -0
  8. package/dist/artifacts.js +159 -0
  9. package/dist/artifacts.js.map +1 -0
  10. package/dist/build.d.ts +20 -0
  11. package/dist/build.js +103 -0
  12. package/dist/build.js.map +1 -0
  13. package/dist/capabilities.d.ts +7 -0
  14. package/dist/capabilities.js +116 -0
  15. package/dist/capabilities.js.map +1 -0
  16. package/dist/catalog-store.d.ts +76 -0
  17. package/dist/catalog-store.js +463 -0
  18. package/dist/catalog-store.js.map +1 -0
  19. package/dist/concurrency.d.ts +1 -0
  20. package/dist/concurrency.js +15 -0
  21. package/dist/concurrency.js.map +1 -0
  22. package/dist/consumer-index.d.ts +7 -29
  23. package/dist/consumer-index.js +24 -15
  24. package/dist/consumer-index.js.map +1 -1
  25. package/dist/digest.d.ts +13 -0
  26. package/dist/digest.js +47 -0
  27. package/dist/digest.js.map +1 -0
  28. package/dist/fs-atomic.d.ts +7 -0
  29. package/dist/fs-atomic.js +30 -0
  30. package/dist/fs-atomic.js.map +1 -0
  31. package/dist/harness.js +2 -1
  32. package/dist/harness.js.map +1 -1
  33. package/dist/icon-mirror.d.ts +5 -4
  34. package/dist/icon-mirror.js +73 -72
  35. package/dist/icon-mirror.js.map +1 -1
  36. package/dist/icons.d.ts +2 -2
  37. package/dist/icons.js +4 -12
  38. package/dist/icons.js.map +1 -1
  39. package/dist/index.d.ts +16 -309
  40. package/dist/index.js +12 -1900
  41. package/dist/index.js.map +1 -1
  42. package/dist/integrity-manifest.d.ts +43 -13
  43. package/dist/integrity-manifest.js +28 -17
  44. package/dist/integrity-manifest.js.map +1 -1
  45. package/dist/json-values.d.ts +3 -0
  46. package/dist/json-values.js +10 -0
  47. package/dist/json-values.js.map +1 -0
  48. package/dist/plugin-release.d.ts +38 -0
  49. package/dist/plugin-release.js +321 -0
  50. package/dist/plugin-release.js.map +1 -0
  51. package/dist/plugin-skips.d.ts +19 -0
  52. package/dist/plugin-skips.js +65 -0
  53. package/dist/plugin-skips.js.map +1 -0
  54. package/dist/policy.d.ts +13 -0
  55. package/dist/policy.js +26 -0
  56. package/dist/policy.js.map +1 -0
  57. package/dist/portable-catalog.d.ts +4 -4
  58. package/dist/portable-catalog.js +14 -126
  59. package/dist/portable-catalog.js.map +1 -1
  60. package/dist/portable-verify.d.ts +2 -0
  61. package/dist/portable-verify.js +68 -0
  62. package/dist/portable-verify.js.map +1 -0
  63. package/dist/publication.d.ts +12 -12
  64. package/dist/publication.js +31 -21
  65. package/dist/publication.js.map +1 -1
  66. package/dist/published-output.d.ts +17 -0
  67. package/dist/published-output.js +49 -0
  68. package/dist/published-output.js.map +1 -0
  69. package/dist/site-icons.d.ts +4 -3
  70. package/dist/site-icons.js +38 -11
  71. package/dist/site-icons.js.map +1 -1
  72. package/dist/snapshots.d.ts +39 -0
  73. package/dist/snapshots.js +238 -0
  74. package/dist/snapshots.js.map +1 -0
  75. package/dist/source-resolution.d.ts +28 -0
  76. package/dist/source-resolution.js +158 -0
  77. package/dist/source-resolution.js.map +1 -0
  78. package/dist/sources.d.ts +5 -1
  79. package/dist/sources.js +21 -27
  80. package/dist/sources.js.map +1 -1
  81. package/dist/synchronize.d.ts +44 -0
  82. package/dist/synchronize.js +118 -0
  83. package/dist/synchronize.js.map +1 -0
  84. package/dist/taxonomy.d.ts +23 -0
  85. package/dist/taxonomy.js +62 -0
  86. package/dist/taxonomy.js.map +1 -0
  87. package/dist/upstream.d.ts +42 -0
  88. package/dist/upstream.js +179 -0
  89. package/dist/upstream.js.map +1 -0
  90. package/dist/verify-artifacts.d.ts +36 -0
  91. package/dist/verify-artifacts.js +286 -0
  92. package/dist/verify-artifacts.js.map +1 -0
  93. package/native/build-provenance.txt +5 -5
  94. package/native/staged-reader-win32-arm64.node +0 -0
  95. package/native/staged-reader-win32-x64.node +0 -0
  96. package/package.json +3 -3
  97. package/src/agent-plugins-catalog.test.ts +2 -2
  98. package/src/agent-plugins-catalog.ts +46 -49
  99. package/src/agent-plugins-schema.ts +11 -1
  100. package/src/artifacts.test.ts +80 -0
  101. package/src/artifacts.ts +234 -0
  102. package/src/build.ts +131 -0
  103. package/src/capabilities.ts +153 -0
  104. package/src/catalog-store.ts +519 -0
  105. package/src/concurrency.ts +19 -0
  106. package/src/consumer-index.test.ts +147 -16
  107. package/src/consumer-index.ts +35 -43
  108. package/src/default-head.test.ts +143 -0
  109. package/src/digest.ts +55 -0
  110. package/src/fs-atomic.ts +32 -0
  111. package/src/harness.test.ts +4 -32
  112. package/src/harness.ts +2 -1
  113. package/src/icon-mirror.ts +78 -72
  114. package/src/icons.test.ts +0 -4
  115. package/src/icons.ts +4 -4
  116. package/src/index.test.ts +172 -46
  117. package/src/index.ts +58 -2562
  118. package/src/integrity-manifest.ts +41 -21
  119. package/src/json-values.ts +11 -0
  120. package/src/manifest-retry.test.ts +49 -0
  121. package/src/plugin-release.ts +457 -0
  122. package/src/plugin-skips.ts +110 -0
  123. package/src/policy.ts +39 -0
  124. package/src/portable-catalog.test.ts +0 -5
  125. package/src/portable-catalog.ts +16 -143
  126. package/src/portable-verify.ts +71 -0
  127. package/src/publication.test.ts +29 -1
  128. package/src/publication.ts +38 -22
  129. package/src/published-output.ts +60 -0
  130. package/src/site-icons.test.ts +73 -1
  131. package/src/site-icons.ts +31 -14
  132. package/src/snapshots.ts +287 -0
  133. package/src/source-config-sync.test.ts +84 -0
  134. package/src/source-digest.test.ts +61 -8
  135. package/src/source-resolution.ts +233 -0
  136. package/src/sources-json.test.ts +18 -2
  137. package/src/sources.ts +27 -33
  138. package/src/synchronize.ts +213 -0
  139. package/src/taxonomy.ts +87 -0
  140. package/src/upstream.ts +194 -0
  141. package/src/verify-artifacts.ts +365 -0
  142. package/src/write-artifacts.test.ts +374 -0
  143. package/dist/source-digest.d.ts +0 -2
  144. package/dist/source-digest.js +0 -12
  145. package/dist/source-digest.js.map +0 -1
  146. package/native/README.md +0 -24
  147. package/src/source-digest.ts +0 -14
@@ -32,7 +32,6 @@ function fetcher(site: Record<string, string | Uint8Array>): SiteIconFetcher & {
32
32
  return {
33
33
  textCalls,
34
34
  byteCalls,
35
- digest: (bytes) => byteDigest(bytes),
36
35
  fetchText: async (url) => {
37
36
  textCalls.push(url)
38
37
  const value = site[url]
@@ -96,6 +95,68 @@ describe('site icon discovery', () => {
96
95
  ])
97
96
  })
98
97
 
98
+ test('resolves relative icon links against the first document base URL', () => {
99
+ const baseHtml = `<!-- <base href="https://ignored.example/"> <script> -->
100
+ <script>const sample = '<!-- <base href="https://script.example/">'</script>
101
+ <meta content="<base href='https://attribute.example/'>">
102
+ <base target="_blank"><base href="/assets/icons/">
103
+ <base href="https://later.example/">
104
+ <link rel="icon" href="mark.png">`
105
+ expect(discoverSiteIcons(baseHtml, 'https://vendor.example/docs/page')).toEqual([
106
+ 'https://vendor.example/assets/icons/mark.png',
107
+ 'https://vendor.example/favicon.ico',
108
+ ])
109
+ })
110
+
111
+ test('ignores bases and links inside text and template content', () => {
112
+ for (const inert of [
113
+ '<title><base href="https://title.example/"></title>',
114
+ '<textarea><base href="https://textarea.example/"></textarea>',
115
+ `<template><template><base href="https://nested.example/"></template>
116
+ <link rel="icon" href="https://template.example/icon.png"></template>`,
117
+ ]) {
118
+ expect(
119
+ discoverSiteIcons(
120
+ `${inert}<base href="/real/"><link rel="icon" href="mark.png">`,
121
+ 'https://vendor.example/'
122
+ )
123
+ ).toEqual(['https://vendor.example/real/mark.png', 'https://vendor.example/favicon.ico'])
124
+ }
125
+ })
126
+
127
+ test('decodes HTML character references in URLs', () => {
128
+ expect(
129
+ discoverSiteIcons(
130
+ '<base href="/icons/?a=1&amp;b=2"><link rel="icon" href="mark.png?x=1&amp;y=2">',
131
+ 'https://vendor.example/'
132
+ )
133
+ ).toEqual([
134
+ 'https://vendor.example/icons/mark.png?x=1&y=2',
135
+ 'https://vendor.example/favicon.ico',
136
+ ])
137
+ })
138
+
139
+ test('retains the homepage fallback and URL policy with a document base', () => {
140
+ expect(
141
+ discoverSiteIcons(
142
+ '<base href="https://cdn.example/icons/"><link rel="icon" href="mark.png">',
143
+ 'https://vendor.example/docs/page'
144
+ )
145
+ ).toEqual(['https://cdn.example/icons/mark.png', 'https://vendor.example/favicon.ico'])
146
+ expect(
147
+ discoverSiteIcons(
148
+ '<base href="http://vendor.example/assets/"><link rel="icon" href="mark.png">',
149
+ 'https://vendor.example/docs/page'
150
+ )
151
+ ).toEqual(['https://vendor.example/favicon.ico'])
152
+ expect(
153
+ discoverSiteIcons(
154
+ '<base href="http://["><base href="/later/"><link rel="icon" href="mark.png">',
155
+ 'https://vendor.example/docs/page'
156
+ )
157
+ ).toEqual(['https://vendor.example/docs/mark.png', 'https://vendor.example/favicon.ico'])
158
+ })
159
+
99
160
  test('prefers the larger declared size within a group', () => {
100
161
  const sizes = `<link rel="icon" sizes="32x32" href="/small.png">
101
162
  <link rel="icon" sizes="512x512" href="/large.png">`
@@ -163,6 +224,17 @@ describe('site icon resolution', () => {
163
224
  expect(io.byteCalls).toEqual(['https://vendor.example/apple-icon.png'])
164
225
  })
165
226
 
227
+ test('fetches a relative icon from the document base', async () => {
228
+ const page = 'https://vendor.example/docs/page'
229
+ const baseHtml = '<head><base href="/assets/"><link rel="icon" href="mark.png"></head>'
230
+ const io = fetcher({ [page]: baseHtml, 'https://vendor.example/assets/mark.png': PNG })
231
+ expect(await resolveSiteIcon(page, io)).toMatchObject({
232
+ url: 'https://vendor.example/assets/mark.png',
233
+ homepage: page,
234
+ })
235
+ expect(io.byteCalls).toEqual(['https://vendor.example/assets/mark.png'])
236
+ })
237
+
166
238
  test('accepts an icon container and skips a candidate that fails inspection', async () => {
167
239
  const ico = fetcher({ [homepage]: html, 'https://vendor.example/apple-icon.png': ICO })
168
240
  expect(await resolveSiteIcon(homepage, ico)).toMatchObject({ contentType: 'image/x-icon' })
package/src/site-icons.ts CHANGED
@@ -1,3 +1,5 @@
1
+ import { parseFragment, type DefaultTreeAdapterTypes } from 'parse5'
2
+ import { byteDigest } from './digest.js'
1
3
  import { inspectIconBytes, MAX_ICON_BYTES, type IconContentType } from './icons.js'
2
4
 
3
5
  /**
@@ -5,8 +7,8 @@ import { inspectIconBytes, MAX_ICON_BYTES, type IconContentType } from './icons.
5
7
  *
6
8
  * Many plugins are a thin wrapper over a vendor's API: the content carries no
7
9
  * logo, and the declared homepage is the only pointer to the vendor. Fetching
8
- * that site's icon once at publish time, verifying it and mirroring it as a
9
- * release asset replaces a per-render favicon lookup in the client.
10
+ * that site's icon once at publish time, verifying it and mirroring it beside
11
+ * the catalog replaces a per-render favicon lookup in the client.
10
12
  */
11
13
  export const SITE_ICON_MAX_HTML_BYTES = 4 * 1024 * 1024
12
14
 
@@ -76,19 +78,35 @@ export function isPublicHostname(hostname: string): boolean {
76
78
  * `apple-touch-icon` is preferred because it is a square PNG sized for a home
77
79
  * screen; declared `icon` links follow in declared order; `/favicon.ico` is the
78
80
  * conventional last resort when nothing is declared.
81
+ * Relative links use the document's first base href when present; the fallback
82
+ * stays on the homepage origin.
79
83
  */
80
84
  export function discoverSiteIcons(html: string, baseUrl: string): readonly string[] {
81
85
  const candidates: { href: string; rank: number }[] = []
82
- const linkTags = html.match(/<link(?=[ \t\r\n\f/>])(?:"[^"]*"|'[^']*'|[^'">])*>/gi) ?? []
83
- for (const tag of linkTags) {
84
- const attributes = new Map<string, string>()
85
- for (const attribute of tag.matchAll(
86
- /\s+([^\s"'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g
87
- )) {
88
- const name = attribute[1]!.toLowerCase()
89
- if (!attributes.has(name))
90
- attributes.set(name, attribute[2] ?? attribute[3] ?? attribute[4] ?? '')
86
+ const elements: DefaultTreeAdapterTypes.Element[] = []
87
+ const nodes: DefaultTreeAdapterTypes.ChildNode[] = parseFragment(html).childNodes.toReversed()
88
+ while (nodes.length > 0) {
89
+ const node = nodes.pop()!
90
+ if (!('tagName' in node)) continue
91
+ if (node.namespaceURI === 'http://www.w3.org/1999/xhtml') elements.push(node)
92
+ // Template contents are a separate, inert document fragment.
93
+ if (node.tagName !== 'template') nodes.push(...node.childNodes.toReversed())
94
+ }
95
+ let resolutionBase = baseUrl
96
+ for (const element of elements) {
97
+ if (element.tagName !== 'base') continue
98
+ const attributes = new Map(element.attrs.map(({ name, value }) => [name, value]))
99
+ if (!attributes.has('href')) continue
100
+ try {
101
+ resolutionBase = new URL(attributes.get('href')!, baseUrl).toString()
102
+ } catch {
103
+ // An invalid first base falls back to the document URL.
91
104
  }
105
+ break
106
+ }
107
+ for (const element of elements) {
108
+ if (element.tagName !== 'link') continue
109
+ const attributes = new Map(element.attrs.map(({ name, value }) => [name, value]))
92
110
  const rel = new Set((attributes.get('rel') ?? '').toLowerCase().split(/\s+/))
93
111
  if (
94
112
  !['apple-touch-icon', 'apple-touch-icon-precomposed', 'icon', 'mask-icon'].some((name) =>
@@ -112,7 +130,7 @@ export function discoverSiteIcons(html: string, baseUrl: string): readonly strin
112
130
  const resolved: { url: string; rank: number }[] = []
113
131
  for (const candidate of candidates.toSorted((left, right) => left.rank - right.rank)) {
114
132
  try {
115
- const url = new URL(candidate.href, baseUrl)
133
+ const url = new URL(candidate.href, resolutionBase)
116
134
  if (url.protocol !== 'https:' || !isPublicHostname(url.hostname)) continue
117
135
  resolved.push({ url: url.toString(), rank: candidate.rank })
118
136
  } catch {
@@ -134,7 +152,6 @@ export function discoverSiteIcons(html: string, baseUrl: string): readonly strin
134
152
  export interface SiteIconFetcher {
135
153
  readonly fetchText: (url: string) => Promise<string>
136
154
  readonly fetchBytes: (url: string) => Promise<Uint8Array>
137
- readonly digest: (bytes: Uint8Array) => string
138
155
  }
139
156
 
140
157
  /**
@@ -171,7 +188,7 @@ export async function resolveSiteIcon(
171
188
  homepage,
172
189
  bytes,
173
190
  contentType: inspected.contentType,
174
- digest: fetcher.digest(bytes),
191
+ digest: byteDigest(bytes),
175
192
  }
176
193
  }
177
194
  return { failed: 'NO_USABLE_ICON' }
@@ -0,0 +1,287 @@
1
+ import { promises as fs } from 'node:fs'
2
+ import { join, relative, resolve } from 'node:path'
3
+ import { Readable } from 'node:stream'
4
+ import { createGunzip } from 'node:zlib'
5
+ import { extract } from 'tar-stream'
6
+ import type { CatalogPolicy } from './policy.js'
7
+ import { PluginSafetyError } from './plugin-skips.js'
8
+ import { canonicalRepositoryUrl, safeRelativePath, type SourceConfig } from './sources.js'
9
+ import { fetchUpstream, GITHUB_REPOSITORY_HOSTS, upstreamRequestHeaders } from './upstream.js'
10
+
11
+ export interface Snapshot {
12
+ readonly files: ReadonlyMap<string, Uint8Array>
13
+ readonly symlinks: readonly string[]
14
+ }
15
+
16
+ export interface SnapshotLoader {
17
+ load(repositoryUrl: string, commitSha: string, pluginSubdirectory: string): Promise<Snapshot>
18
+ }
19
+
20
+ /**
21
+ * Ceiling on bytes read out of one repository archive.
22
+ *
23
+ * The per-plugin policy still bounds what any single plugin may contain; this
24
+ * only stops a decompression bomb from exhausting the build host, and it is
25
+ * enforced while the stream is read rather than after it is buffered.
26
+ */
27
+ const MAX_ARCHIVE_BYTES = 512 * 1024 * 1024
28
+
29
+ function concatChunks(chunks: readonly Uint8Array[], length: number): Uint8Array {
30
+ const out = new Uint8Array(length)
31
+ let offset = 0
32
+ for (const chunk of chunks) {
33
+ out.set(chunk, offset)
34
+ offset += chunk.byteLength
35
+ }
36
+ return out
37
+ }
38
+
39
+ /**
40
+ * Read one GitHub repository tarball into path -> bytes.
41
+ *
42
+ * GitHub wraps every entry in a single top-level `<repo>-<sha>/` directory,
43
+ * which is stripped so the keys match the tree API's repository-relative
44
+ * paths. Only regular files are taken: directories, symlinks, hardlinks and
45
+ * the `pax_global_header` record are skipped, so a link can never be
46
+ * materialized as content. Every remaining path goes through
47
+ * `safeRelativePath`, so an archive that tries to escape the repository is
48
+ * rejected rather than partially trusted. This is a whole-repository archive
49
+ * shared across plugins, so an oversized unrelated file is drained and
50
+ * counted against the archive budget but not retained. The selected plugin's
51
+ * files are checked against the tree before the archive is fetched and again
52
+ * when its snapshot is validated.
53
+ */
54
+ export async function readArchiveFiles(
55
+ archive: Uint8Array,
56
+ policy: CatalogPolicy
57
+ ): Promise<Map<string, Uint8Array>> {
58
+ const files = new Map<string, Uint8Array>()
59
+ let totalBytes = 0
60
+ const parser = extract()
61
+ const gunzip = createGunzip()
62
+ const settled = new Promise<void>((resolveDone, rejectDone) => {
63
+ const fail = (error: unknown): void => {
64
+ gunzip.destroy()
65
+ parser.destroy()
66
+ rejectDone(error)
67
+ }
68
+ parser.on('entry', (header, stream, next) => {
69
+ const skip = (): void => {
70
+ stream.on('end', next)
71
+ stream.resume()
72
+ }
73
+ if (header.type !== 'file') return skip()
74
+ const segments = header.name.split('/')
75
+ if (segments.length < 2) return skip()
76
+ let path: string
77
+ try {
78
+ path = safeRelativePath(segments.slice(1).join('/'), 'TREE_PATH_INVALID')
79
+ } catch (error) {
80
+ return fail(error)
81
+ }
82
+ const chunks: Uint8Array[] = []
83
+ let length = 0
84
+ stream.on('data', (chunk: unknown) => {
85
+ const bytes = chunk as Uint8Array
86
+ length += bytes.byteLength
87
+ if (totalBytes + length > MAX_ARCHIVE_BYTES) {
88
+ fail(new Error('PLUGIN_SIZE_POLICY: archive'))
89
+ return
90
+ }
91
+ if (length <= policy.maxFileBytes) chunks.push(bytes)
92
+ else chunks.length = 0
93
+ })
94
+ stream.on('error', fail)
95
+ stream.on('end', () => {
96
+ try {
97
+ totalBytes += length
98
+ if (length <= policy.maxFileBytes) files.set(path, concatChunks(chunks, length))
99
+ next()
100
+ } catch (error) {
101
+ fail(error)
102
+ }
103
+ })
104
+ stream.resume()
105
+ })
106
+ parser.on('error', fail)
107
+ gunzip.on('error', fail)
108
+ parser.on('finish', () => resolveDone())
109
+ })
110
+ Readable.from([archive]).pipe(gunzip).pipe(parser)
111
+ await settled
112
+ return files
113
+ }
114
+
115
+ export class NetworkSnapshotLoader implements SnapshotLoader {
116
+ readonly #policy: CatalogPolicy
117
+ readonly #trees = new Map<string, Promise<readonly TreeEntry[]>>()
118
+ readonly #archives = new Map<string, Promise<ReadonlyMap<string, Uint8Array>>>()
119
+
120
+ constructor(policy: CatalogPolicy) {
121
+ this.#policy = policy
122
+ }
123
+
124
+ async load(
125
+ repositoryUrl: string,
126
+ commitSha: string,
127
+ pluginSubdirectory: string
128
+ ): Promise<Snapshot> {
129
+ const parsed = new URL(canonicalRepositoryUrl(repositoryUrl))
130
+ if (!GITHUB_REPOSITORY_HOSTS.includes(parsed.hostname))
131
+ throw new Error(`GIT_HOST_UNSUPPORTED: ${parsed.hostname}`)
132
+ const [owner, repo] = parsed.pathname.split('/').filter(Boolean)
133
+ if (!owner || !repo) throw new Error('GIT_REPOSITORY_INVALID')
134
+ const key = `${owner}/${repo}@${commitSha}`
135
+ let pendingTree = this.#trees.get(key)
136
+ if (!pendingTree) {
137
+ pendingTree = (async () => {
138
+ const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${commitSha}?recursive=1`
139
+ const response = await fetchUpstream(treeUrl, {
140
+ headers: upstreamRequestHeaders(treeUrl),
141
+ })
142
+ if (!response.ok)
143
+ throw new PluginSafetyError('PLUGIN_SOURCE_UNAVAILABLE', [`${owner}/${repo}`])
144
+ const payload = (await response.json()) as { truncated?: boolean; tree?: TreeEntry[] }
145
+ if (payload.truncated) throw new Error(`GIT_TREE_TRUNCATED: ${owner}/${repo}@${commitSha}`)
146
+ return (payload.tree ?? []).map((entry) => ({
147
+ ...entry,
148
+ path: safeRelativePath(entry.path, 'TREE_PATH_INVALID'),
149
+ }))
150
+ })().catch((error: unknown) => {
151
+ this.#trees.delete(key)
152
+ throw error
153
+ })
154
+ this.#trees.set(key, pendingTree)
155
+ }
156
+ const tree = await pendingTree
157
+ const prefix = pluginSubdirectory ? `${safeRelativePath(pluginSubdirectory)}/` : ''
158
+ const submodules = tree.filter(
159
+ (entry) =>
160
+ entry.path.startsWith(prefix) && (entry.type === 'commit' || entry.mode === '160000')
161
+ )
162
+ if (submodules.length > 0)
163
+ throw new Error(`GIT_SUBMODULE_UNSUPPORTED: ${repositoryUrl}:${pluginSubdirectory}`)
164
+ const selected = tree.filter((entry) => entry.type === 'blob' && entry.path.startsWith(prefix))
165
+ if (selected.length === 0)
166
+ throw new Error(`PLUGIN_PATH_NOT_FOUND: ${repositoryUrl}:${pluginSubdirectory}`)
167
+ const files = new Map<string, Uint8Array>()
168
+ const symlinks = tree
169
+ .filter((entry) => entry.mode === '120000' && entry.path.startsWith(prefix))
170
+ .map((entry) => entry.path)
171
+ if (selected.length > this.#policy.maxFilesPerPlugin) throw new Error('PLUGIN_TOO_MANY_FILES')
172
+ let totalSize = 0
173
+ for (const entry of selected) {
174
+ const size = entry.size ?? 0
175
+ if (size > this.#policy.maxFileBytes || totalSize + size > this.#policy.maxBytesPerPlugin)
176
+ throw new Error(`PLUGIN_SIZE_POLICY: ${entry.path}`)
177
+ totalSize += size
178
+ }
179
+ // The tree above already bounded this plugin before a byte was pulled, so
180
+ // the archive is the only unbounded read left and it is capped while it
181
+ // streams. One archive per repository replaces one request per file.
182
+ const archive = await this.#archiveFiles(key, owner, repo, commitSha)
183
+ for (const entry of selected) {
184
+ const bytes = archive.get(entry.path)
185
+ if (!bytes) throw new PluginSafetyError('PLUGIN_FILE_UNAVAILABLE', [entry.path])
186
+ if (bytes.byteLength > this.#policy.maxFileBytes)
187
+ throw new Error(`PLUGIN_FILE_TOO_LARGE: ${entry.path}`)
188
+ files.set(
189
+ prefix ? relative(prefix.slice(0, -1), entry.path).replaceAll('\\', '/') : entry.path,
190
+ bytes
191
+ )
192
+ }
193
+ return { files, symlinks }
194
+ }
195
+
196
+ async #archiveFiles(
197
+ key: string,
198
+ owner: string,
199
+ repo: string,
200
+ commitSha: string
201
+ ): Promise<ReadonlyMap<string, Uint8Array>> {
202
+ const cached = this.#archives.get(key)
203
+ if (cached) return cached
204
+ const pending = (async () => {
205
+ const url = `https://api.github.com/repos/${owner}/${repo}/tarball/${commitSha}`
206
+ const response = await fetchUpstream(url, { headers: upstreamRequestHeaders(url) })
207
+ if (!response.ok)
208
+ throw new PluginSafetyError('PLUGIN_SOURCE_UNAVAILABLE', [`${owner}/${repo}`])
209
+ return readArchiveFiles(new Uint8Array(await response.arrayBuffer()), this.#policy)
210
+ })().catch((error: unknown) => {
211
+ this.#archives.delete(key)
212
+ throw error
213
+ })
214
+ this.#archives.set(key, pending)
215
+ return pending
216
+ }
217
+ }
218
+
219
+ interface TreeEntry {
220
+ readonly path: string
221
+ readonly mode?: string
222
+ readonly type?: string
223
+ readonly size?: number
224
+ }
225
+
226
+ export class FixtureSnapshotLoader implements SnapshotLoader {
227
+ readonly #root: string
228
+ readonly #sources: ReadonlyMap<string, SourceConfig>
229
+
230
+ constructor(root: string, sources: readonly SourceConfig[]) {
231
+ this.#root = resolve(root)
232
+ this.#sources = new Map(
233
+ sources.map((source) => [canonicalRepositoryUrl(source.repositoryUrl), source])
234
+ )
235
+ }
236
+
237
+ async load(
238
+ repositoryUrl: string,
239
+ _commitSha: string,
240
+ pluginSubdirectory: string
241
+ ): Promise<Snapshot> {
242
+ const source = this.#sources.get(canonicalRepositoryUrl(repositoryUrl))
243
+ if (!source?.fixturePath) throw new Error(`FIXTURE_REPOSITORY_NOT_FOUND: ${repositoryUrl}`)
244
+ return snapshotFromDirectory(join(this.#root, source.fixturePath), pluginSubdirectory)
245
+ }
246
+ }
247
+
248
+ export class DirectorySnapshotLoader implements SnapshotLoader {
249
+ async load(
250
+ _repositoryUrl: string,
251
+ _commitSha: string,
252
+ pluginSubdirectory: string
253
+ ): Promise<Snapshot> {
254
+ throw new Error(`DIRECTORY_LOADER_REQUIRES_ROOT: ${pluginSubdirectory}`)
255
+ }
256
+ }
257
+
258
+ export async function snapshotFromDirectory(
259
+ root: string,
260
+ pluginSubdirectory: string
261
+ ): Promise<Snapshot> {
262
+ const directory = resolve(root, pluginSubdirectory)
263
+ const files = new Map<string, Uint8Array>()
264
+ const symlinks: string[] = []
265
+ const rootStat = await fs.lstat(directory)
266
+ if (rootStat.isSymbolicLink()) return { files, symlinks: [pluginSubdirectory || '.'] }
267
+ await walk(directory, directory, files, symlinks)
268
+ return { files, symlinks }
269
+ }
270
+
271
+ async function walk(
272
+ root: string,
273
+ current: string,
274
+ files: Map<string, Uint8Array>,
275
+ symlinks: string[]
276
+ ): Promise<void> {
277
+ for (const entry of await fs.readdir(current, { withFileTypes: true })) {
278
+ const absolute = join(current, entry.name)
279
+ const relativePath = safeRelativePath(relative(root, absolute), 'DIRECTORY_PATH_INVALID')
280
+ if (entry.isSymbolicLink()) {
281
+ symlinks.push(relativePath)
282
+ continue
283
+ }
284
+ if (entry.isDirectory()) await walk(root, absolute, files, symlinks)
285
+ else if (entry.isFile()) files.set(relativePath, await fs.readFile(absolute))
286
+ }
287
+ }
@@ -0,0 +1,84 @@
1
+ import { copyFile } from 'node:fs/promises'
2
+ import { join } from 'node:path'
3
+ import { expect, test } from 'bun:test'
4
+ import { synchronizePortable } from './portable-catalog.js'
5
+ import { withPortableFixture } from '../test-support/portable-fixture.js'
6
+
7
+ test('rebuilds and reports source metadata changes when source commits stay fixed', () =>
8
+ withPortableFixture(async (input, root) => {
9
+ const first = await synchronizePortable(input)
10
+ const old = input.sources[0]!
11
+ await copyFile(
12
+ join(root, 'fixtures/source/marketplace.json'),
13
+ join(root, 'fixtures/source/alternate.json')
14
+ )
15
+ const cases = [
16
+ { ...old, displayName: 'Renamed source' },
17
+ { ...old, defaultBranch: 'stable' },
18
+ { ...old, manifestPath: 'alternate.json' },
19
+ { ...old, marketplaceDialect: 'openai' as const },
20
+ ]
21
+ for (const source of cases) {
22
+ const result = await synchronizePortable({
23
+ ...input,
24
+ sources: [source],
25
+ existingLock: first.lock!,
26
+ existingCatalog: first.catalog!,
27
+ })
28
+ expect(result.changed).toBe(true)
29
+ expect(result.changeReport.changedSources).toContain(source.sourceId)
30
+ expect(result.catalog?.sources[0]?.displayName).toBe(source.displayName)
31
+ expect(result.lock?.sources[0]?.defaultBranch).toBe(source.defaultBranch)
32
+ expect(result.lock?.sources[0]?.manifestPath).toBe(source.manifestPath)
33
+ expect(result.lock?.sources[0]?.marketplaceDialect).toBe(source.marketplaceDialect)
34
+ const repeated = await synchronizePortable({
35
+ ...input,
36
+ sources: [source],
37
+ existingLock: result.lock!,
38
+ existingCatalog: result.catalog!,
39
+ })
40
+ expect(repeated.changed).toBe(false)
41
+ }
42
+ const equivalent = await synchronizePortable({
43
+ ...input,
44
+ sources: [{ ...old, repositoryUrl: `${old.repositoryUrl}.git` }],
45
+ existingLock: first.lock!,
46
+ existingCatalog: first.catalog!,
47
+ })
48
+ expect(equivalent.changed).toBe(false)
49
+ }))
50
+
51
+ test('rebuilds if the marketplace repository changes while external plugin pins stay equal', () =>
52
+ withPortableFixture(async (input, root) => {
53
+ const { writeFile } = await import('node:fs/promises')
54
+ await writeFile(
55
+ join(root, 'fixtures/source/marketplace.json'),
56
+ JSON.stringify({
57
+ plugins: [
58
+ { name: 'remote', source: { source: 'git', url: 'https://github.com/example/external' } },
59
+ ],
60
+ })
61
+ )
62
+ const snapshotLoader = {
63
+ load: async () => ({
64
+ files: new Map([['plugin.json', new TextEncoder().encode('{"name":"Remote"}')]]),
65
+ symlinks: [],
66
+ }),
67
+ }
68
+ const options = { ...input, snapshotLoader }
69
+ const first = await synchronizePortable(options)
70
+ const source = {
71
+ ...input.sources[0]!,
72
+ repositoryUrl: 'https://github.com/example/moved-marketplace',
73
+ }
74
+ const result = await synchronizePortable({
75
+ ...options,
76
+ sources: [source],
77
+ existingLock: first.lock!,
78
+ })
79
+ expect(result.changed).toBe(true)
80
+ expect(result.lock?.sources[0]?.repositoryUrl).toBe(source.repositoryUrl)
81
+ expect(result.catalog?.sources[0]?.repositoryUrl).toBe(source.repositoryUrl)
82
+ expect(result.changeReport.changedSources).toEqual([source.sourceId])
83
+ expect(result.lock?.sources[0]?.pluginPins).toEqual(first.lock?.sources[0]?.pluginPins)
84
+ }))
@@ -1,4 +1,4 @@
1
- import { mkdtemp, rm, writeFile } from 'node:fs/promises'
1
+ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
2
2
  import { join } from 'node:path'
3
3
  import { spawnSync } from 'node:child_process'
4
4
  import { pathToFileURL } from 'node:url'
@@ -11,20 +11,17 @@ test('source digests agree across consumer locales and equivalent Unicode path o
11
11
  try {
12
12
  const entry = join(directory, 'entry.ts')
13
13
  const output = join(directory, 'consumer.mjs')
14
- await writeFile(
15
- entry,
16
- `export { bytesDigest } from '../index.ts'\nexport { sourceDigest } from '../agent-plugins-catalog.ts'\n`
17
- )
14
+ await writeFile(entry, `export { sourceDigest } from '../digest.ts'\n`)
18
15
  const build = await Bun.build({ entrypoints: [entry], target: 'node', packages: 'external' })
19
16
  expect(build.success).toBe(true)
20
17
  await Bun.write(output, build.outputs[0]!)
21
18
  const script = `
22
- import { bytesDigest, sourceDigest } from ${JSON.stringify(pathToFileURL(output).href)};
19
+ import { sourceDigest } from ${JSON.stringify(pathToFileURL(output).href)};
23
20
  const paths = ['z.txt', 'ä.txt', 'é.txt', 'e\\u0301.txt'];
24
21
  const entries = paths.map((path) => [path, new TextEncoder().encode(path)]);
25
22
  const forward = new Map(entries), reverse = new Map([...entries].reverse());
26
23
  console.log(JSON.stringify([
27
- bytesDigest(forward), bytesDigest(reverse), sourceDigest(forward), sourceDigest(reverse)
24
+ sourceDigest(forward), sourceDigest(reverse)
28
25
  ]));
29
26
  `
30
27
  const results = ['en_US.UTF-8', 'sv_SE.UTF-8'].map((locale) => {
@@ -34,7 +31,6 @@ test('source digests agree across consumer locales and equivalent Unicode path o
34
31
  })
35
32
  if (result.error) throw result.error
36
33
  if (result.status !== 0) throw new Error(result.stderr)
37
- expect(result.status).toBe(0)
38
34
  return JSON.parse(result.stdout) as string[]
39
35
  })
40
36
  expect(new Set(results.flat()).size).toBe(1)
@@ -42,3 +38,60 @@ test('source digests agree across consumer locales and equivalent Unicode path o
42
38
  await rm(directory, { recursive: true, force: true })
43
39
  }
44
40
  })
41
+
42
+ test('source lock identity is independent of the consumer locale', async () => {
43
+ const directory = await mkdtemp(join(import.meta.dirname, '.source-lock-'))
44
+ try {
45
+ const source = join(directory, 'fixture', 'source')
46
+ await mkdir(source, { recursive: true })
47
+ await writeFile(
48
+ join(source, 'marketplace.json'),
49
+ JSON.stringify({
50
+ plugins: [
51
+ { name: 'a-plugin', source: './a' },
52
+ { name: 'ä-plugin', source: './u' },
53
+ { name: 'z-plugin', source: './z' },
54
+ ],
55
+ })
56
+ )
57
+ const entry = join(directory, 'entry.ts')
58
+ const output = join(directory, 'consumer.mjs')
59
+ await writeFile(entry, "export { synchronize } from '../index.ts'\n")
60
+ const build = await Bun.build({ entrypoints: [entry], target: 'node', packages: 'external' })
61
+ expect(build.success).toBe(true)
62
+ await Bun.write(output, build.outputs[0]!)
63
+ const script = `
64
+ import { synchronize } from ${JSON.stringify(pathToFileURL(output).href)};
65
+ const result = await synchronize({
66
+ mode: 'offline', fixtureRoot: ${JSON.stringify(join(directory, 'fixture'))},
67
+ sources: [{sourceId:'test-source',displayName:'Test',marketplaceDialect:'claude',
68
+ repositoryUrl:'https://github.com/example/plugins',manifestPath:'marketplace.json',
69
+ defaultBranch:'main',trustClassification:'official',fixturePath:'source',
70
+ fixtureCommitSha:'a'.repeat(40)}],
71
+ categoryMap:{aliases:{},fallback:'other'},productAliases:{aliases:{}},
72
+ policy:{allowedRepositoryProtocols:['https:'],allowedRepositoryHosts:['github.com'],
73
+ maxFilesPerPlugin:100,maxBytesPerPlugin:100000,maxFileBytes:20000,
74
+ maxMarketplacePlugins:100,denyExecutableLifecycleScripts:true,
75
+ publishRequiresCompleteContent:false,sensitiveCapabilityTypes:[]},metadataOnly:true
76
+ });
77
+ console.log(JSON.stringify({lockId:result.lock.lockId,pins:result.lock.sources[0].pluginPins}));
78
+ `
79
+ const results = ['en_US.UTF-8', 'sv_SE.UTF-8'].map((locale) => {
80
+ const result = spawnSync('node', ['--input-type=module', '-e', script], {
81
+ encoding: 'utf8',
82
+ env: { ...process.env, LANG: locale, LC_ALL: locale },
83
+ })
84
+ if (result.error) throw result.error
85
+ if (result.status !== 0) throw new Error(result.stderr)
86
+ return JSON.parse(result.stdout) as { lockId: string; pins: { pluginName: string }[] }
87
+ })
88
+ expect(results[0]).toEqual(results[1])
89
+ expect(results[0]!.pins.map((pin) => pin.pluginName)).toEqual([
90
+ 'a-plugin',
91
+ 'z-plugin',
92
+ 'ä-plugin',
93
+ ])
94
+ } finally {
95
+ await rm(directory, { recursive: true, force: true })
96
+ }
97
+ })