@foss.global/codefeed 1.12.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +19 -0
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.forgeinventory.d.ts +21 -0
- package/dist_ts/classes.forgeinventory.js +178 -16
- package/dist_ts/classes.forgeinventoryscanner.d.ts +14 -2
- package/dist_ts/classes.forgeinventoryscanner.js +67 -13
- package/dist_ts/forgeinventory.validation.d.ts +2 -0
- package/dist_ts/forgeinventory.validation.js +15 -1
- package/package.json +2 -2
- package/readme.md +18 -5
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.forgeinventory.ts +173 -17
- package/ts/classes.forgeinventoryscanner.ts +81 -14
- package/ts/forgeinventory.validation.ts +13 -0
|
@@ -29,11 +29,22 @@ export interface IForgeInventoryMetrics {
|
|
|
29
29
|
rateLimitedRequests: number;
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
type
|
|
32
|
+
export type TForgeInventoryVersionInspection =
|
|
33
|
+
| { status: 'observed'; version: string; observation: plugins.fossInterfaces.data.IForgeInventoryObservation }
|
|
34
|
+
| { status: 'unverified'; reason: string; observation: plugins.fossInterfaces.data.IForgeInventoryObservation };
|
|
35
|
+
|
|
36
|
+
type TRepositoryScope = Extract<plugins.fossInterfaces.data.TForgeInventoryScope, { kind: 'repository' }>;
|
|
37
|
+
type TIdentifiedRepositoryScope = TRepositoryScope & { identity: Extract<plugins.fossInterfaces.data.TForgeInventoryIdentity, { status: 'identified' }> };
|
|
38
|
+
type TCollection = 'repositories' | 'namespaces' | 'namespaceDirectory' | 'issues';
|
|
33
39
|
interface IContinuation {
|
|
34
|
-
version:
|
|
40
|
+
version: 3;
|
|
35
41
|
collection: TCollection;
|
|
42
|
+
repositoryKey: string | null;
|
|
43
|
+
/** Endpoint page, reset when a directory moves from organizations to users. */
|
|
36
44
|
page: number;
|
|
45
|
+
/** Completed requests across every endpoint in this collection. */
|
|
46
|
+
pagesRead: number;
|
|
47
|
+
phase: 'organizations' | 'users' | null;
|
|
37
48
|
after: string | null;
|
|
38
49
|
}
|
|
39
50
|
|
|
@@ -88,12 +99,96 @@ export class ForgeInventoryAdapter {
|
|
|
88
99
|
return this.page('namespaces', (item) => this.namespace(item), requestArg);
|
|
89
100
|
}
|
|
90
101
|
|
|
102
|
+
/** Gitea admin organizations/users or GitLab caller-visible group/personal namespaces. */
|
|
103
|
+
public async listNamespaceDirectory(requestArg: IForgeInventoryRequest = {}): Promise<plugins.fossInterfaces.data.IForgeInventoryPage<plugins.fossInterfaces.data.IForgeInventoryNamespace>> {
|
|
104
|
+
if (!this.#options.token) throw new validation.ForgeInventoryError('namespace-directory-requires-token');
|
|
105
|
+
return this.page('namespaceDirectory', (item, state) => this.directoryNamespace(item, state), requestArg);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** All visible open/closed issues in one stable repository, excluding Gitea pull requests. */
|
|
109
|
+
public listIssues(scopeArg: TRepositoryScope, requestArg: IForgeInventoryRequest = {}): Promise<plugins.fossInterfaces.data.IForgeInventoryPage<plugins.fossInterfaces.data.IForgeInventoryIssue>> {
|
|
110
|
+
const scope = this.repositoryScope(scopeArg);
|
|
111
|
+
return this.page('issues', (item) => this.issue(item, scope), requestArg, scope);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** A fresh, bounded observation; failures never retain a previously observed version. */
|
|
115
|
+
public async inspectVersion(requestArg: Omit<IForgeInventoryRequest, 'cursor'> = {}): Promise<TForgeInventoryVersionInspection> {
|
|
116
|
+
const startedAt = new Date().toISOString();
|
|
117
|
+
try {
|
|
118
|
+
const { body } = await this.request(new URL(`${this.#options.baseUrl}/api/${this.#options.kind === 'gitea' ? 'v1' : 'v4'}/version`), requestArg);
|
|
119
|
+
const version = validation.record(body).version;
|
|
120
|
+
// Preserve release suffixes while rejecting arbitrary bodies, URLs and control characters.
|
|
121
|
+
if (typeof version !== 'string' || version.length > 128 || !/^\d+\.\d+\.\d+(?:[-+][A-Za-z0-9][A-Za-z0-9.+-]*)?$/.test(version)) {
|
|
122
|
+
throw new validation.ForgeInventoryError('invalid-server-version');
|
|
123
|
+
}
|
|
124
|
+
return { status: 'observed', version, observation: this.observation(startedAt) };
|
|
125
|
+
} catch (error) {
|
|
126
|
+
return {
|
|
127
|
+
status: 'unverified', observation: this.observation(startedAt),
|
|
128
|
+
reason: error instanceof validation.ForgeInventoryError ? error.message : 'Version inspection failed.',
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
91
133
|
private identity(idNamespace: string, value: unknown): plugins.fossInterfaces.data.TForgeInventoryIdentity {
|
|
92
134
|
const objectId = validation.decimal(value);
|
|
93
135
|
if (objectId === '0') throw new validation.ForgeInventoryError('invalid-identity');
|
|
94
136
|
return { status: 'identified', ref: { instanceId: this.#options.instanceId, idNamespace, objectId, parentObjectId: null } };
|
|
95
137
|
}
|
|
96
138
|
|
|
139
|
+
private repositoryScope(scopeArg: TRepositoryScope): TIdentifiedRepositoryScope {
|
|
140
|
+
const expectedNamespace = this.#options.kind === 'gitea' ? 'gitea.repository.id' : 'gitlab.project.id';
|
|
141
|
+
if (scopeArg?.kind !== 'repository' || scopeArg.identity?.status !== 'identified' ||
|
|
142
|
+
scopeArg.identity.ref.instanceId !== this.#options.instanceId || scopeArg.identity.ref.idNamespace !== expectedNamespace ||
|
|
143
|
+
scopeArg.identity.ref.parentObjectId !== null || typeof scopeArg.identity.ref.objectId !== 'string' ||
|
|
144
|
+
validation.decimal(scopeArg.identity.ref.objectId) === '0' || !Array.isArray(scopeArg.location?.pathSegments) ||
|
|
145
|
+
scopeArg.location.pathSegments.length < 2 || (this.#options.kind === 'gitea' && scopeArg.location.pathSegments.length !== 2)) {
|
|
146
|
+
throw new validation.ForgeInventoryError('invalid-repository-scope');
|
|
147
|
+
}
|
|
148
|
+
const pathSegments = scopeArg.location.pathSegments.map((segment) => {
|
|
149
|
+
if (validation.segments(segment).length !== 1) throw new validation.ForgeInventoryError('invalid-repository-scope');
|
|
150
|
+
return segment;
|
|
151
|
+
});
|
|
152
|
+
return {
|
|
153
|
+
kind: 'repository', identity: { status: 'identified', ref: {
|
|
154
|
+
instanceId: this.#options.instanceId, idNamespace: expectedNamespace, objectId: scopeArg.identity.ref.objectId, parentObjectId: null,
|
|
155
|
+
} },
|
|
156
|
+
location: { pathSegments, webUrl: validation.httpUrl(scopeArg.location.webUrl) },
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
private issue(itemArg: unknown, scope: TIdentifiedRepositoryScope): plugins.fossInterfaces.data.IForgeInventoryIssue {
|
|
161
|
+
const item = validation.record(itemArg);
|
|
162
|
+
const gitea = this.#options.kind === 'gitea';
|
|
163
|
+
if (gitea && item.pull_request !== undefined && item.pull_request !== null) throw new validation.ForgeInventoryError('unexpected-pull-request');
|
|
164
|
+
const repositoryId = validation.decimal(gitea ? validation.record(item.repository).id : item.project_id);
|
|
165
|
+
if (repositoryId !== scope.identity.ref.objectId) throw new validation.ForgeInventoryError('issue-repository-mismatch');
|
|
166
|
+
const number = validation.decimal(gitea ? item.number : item.iid);
|
|
167
|
+
if (number === '0' || typeof item.title !== 'string' || !item.title) throw new validation.ForgeInventoryError('invalid-issue');
|
|
168
|
+
const body = gitea ? item.body : item.description;
|
|
169
|
+
if (body !== undefined && body !== null && typeof body !== 'string') throw new validation.ForgeInventoryError('invalid-issue-body');
|
|
170
|
+
const state = item.state === (gitea ? 'open' : 'opened') ? 'open' : item.state === 'closed' ? 'closed' : null;
|
|
171
|
+
if (state === null && item.state !== undefined && item.state !== null) throw new validation.ForgeInventoryError('invalid-issue-state');
|
|
172
|
+
const actor = gitea ? item.user : item.author;
|
|
173
|
+
let author: plugins.fossInterfaces.data.TForgeInventoryIdentity = { status: 'unresolved', reason: 'Provider author identity was not exposed.' };
|
|
174
|
+
if (actor !== undefined && actor !== null) {
|
|
175
|
+
const actorId = validation.record(actor).id;
|
|
176
|
+
if (gitea && [-1, -2, '-1', '-2'].includes(actorId as number | string)) {
|
|
177
|
+
author = { status: 'unresolved', reason: 'Gitea reports a deleted or built-in author without a regular user identity.' };
|
|
178
|
+
} else if (actorId !== undefined && actorId !== null) {
|
|
179
|
+
author = this.identity(gitea ? 'gitea.user.id' : 'gitlab.user.id', actorId);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return {
|
|
183
|
+
identity: this.identity(gitea ? 'gitea.issue.id' : 'gitlab.issue.id', item.id), repository: scope.identity,
|
|
184
|
+
location: { pathSegments: [...scope.location.pathSegments, ...(gitea ? [] : ['-']), 'issues', number], webUrl: validation.httpUrl(gitea ? item.html_url : item.web_url) },
|
|
185
|
+
number, title: item.title, body: body ?? null, state,
|
|
186
|
+
providerType: gitea ? 'issue' : validation.optionalString(item.issue_type),
|
|
187
|
+
confidential: gitea ? null : validation.optionalBoolean(item.confidential), author,
|
|
188
|
+
createdAt: validation.timestamp(item.created_at), updatedAt: validation.timestamp(item.updated_at), closedAt: validation.timestamp(item.closed_at),
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
|
|
97
192
|
private namespace(itemArg: unknown): plugins.fossInterfaces.data.IForgeInventoryNamespace {
|
|
98
193
|
const item = validation.record(itemArg);
|
|
99
194
|
if (this.#options.kind === 'gitea') {
|
|
@@ -115,6 +210,31 @@ export class ForgeInventoryAdapter {
|
|
|
115
210
|
};
|
|
116
211
|
}
|
|
117
212
|
|
|
213
|
+
private directoryNamespace(itemArg: unknown, state: IContinuation): plugins.fossInterfaces.data.IForgeInventoryNamespace {
|
|
214
|
+
const item = validation.record(itemArg);
|
|
215
|
+
if (this.#options.kind === 'gitea') {
|
|
216
|
+
if (state.phase === 'organizations') return this.namespace(item);
|
|
217
|
+
const pathSegments = validation.segments(item.login);
|
|
218
|
+
if (pathSegments.length !== 1) throw new validation.ForgeInventoryError('invalid-user-namespace-path');
|
|
219
|
+
return {
|
|
220
|
+
identity: this.identity('gitea.user.id', item.id), kind: 'user', parent: null,
|
|
221
|
+
location: { pathSegments, webUrl: validation.httpUrl(item.html_url) },
|
|
222
|
+
visibility: validation.visibility(item.visibility === 'limited' ? 'internal' : item.visibility),
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
if (item.kind === 'group') return this.namespace(item);
|
|
226
|
+
if (item.kind !== 'user') throw new validation.ForgeInventoryError('invalid-namespace-kind');
|
|
227
|
+
const pathSegments = validation.segments(item.full_path);
|
|
228
|
+
if (pathSegments.length !== 1 || (item.parent_id !== undefined && item.parent_id !== null)) {
|
|
229
|
+
throw new validation.ForgeInventoryError('invalid-user-namespace-parent');
|
|
230
|
+
}
|
|
231
|
+
return {
|
|
232
|
+
identity: this.identity('gitlab.namespace.id', item.id), kind: 'user', parent: null,
|
|
233
|
+
location: { pathSegments, webUrl: validation.httpUrl(item.web_url) },
|
|
234
|
+
visibility: validation.visibility(item.visibility),
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
|
|
118
238
|
private repository(itemArg: unknown): plugins.fossInterfaces.data.IForgeInventoryRepository {
|
|
119
239
|
const item = validation.record(itemArg);
|
|
120
240
|
if (this.#options.kind === 'gitea') {
|
|
@@ -146,9 +266,22 @@ export class ForgeInventoryAdapter {
|
|
|
146
266
|
};
|
|
147
267
|
}
|
|
148
268
|
|
|
149
|
-
private url(state: IContinuation): URL {
|
|
269
|
+
private url(state: IContinuation, scope?: TIdentifiedRepositoryScope): URL {
|
|
150
270
|
const gitea = this.#options.kind === 'gitea';
|
|
151
|
-
|
|
271
|
+
if (state.collection === 'issues') {
|
|
272
|
+
if (!scope) throw new validation.ForgeInventoryError('invalid-repository-scope');
|
|
273
|
+
const endpoint = gitea ? `repos/${scope.location.pathSegments.map(encodeURIComponent).join('/')}/issues`
|
|
274
|
+
: `projects/${scope.identity.ref.objectId}/issues`;
|
|
275
|
+
const url = new URL(`${this.#options.baseUrl}/api/${gitea ? 'v1' : 'v4'}/${endpoint}`);
|
|
276
|
+
url.searchParams.set(gitea ? 'limit' : 'per_page', String(this.#options.pageSize));
|
|
277
|
+
url.searchParams.set('page', String(state.page)); url.searchParams.set('state', 'all');
|
|
278
|
+
if (gitea) url.searchParams.set('type', 'issues');
|
|
279
|
+
else { url.searchParams.set('scope', 'all'); url.searchParams.set('order_by', 'created_at'); url.searchParams.set('sort', 'asc'); }
|
|
280
|
+
return url;
|
|
281
|
+
}
|
|
282
|
+
const endpoint = state.collection === 'repositories' ? (gitea ? 'repos/search' : 'projects')
|
|
283
|
+
: state.collection === 'namespaceDirectory' ? (gitea ? `admin/${state.phase === 'organizations' ? 'orgs' : 'users'}` : 'namespaces')
|
|
284
|
+
: (gitea ? 'orgs' : 'groups');
|
|
152
285
|
const url = new URL(`${this.#options.baseUrl}/api/${gitea ? 'v1' : 'v4'}/${endpoint}`);
|
|
153
286
|
url.searchParams.set(gitea ? 'limit' : 'per_page', String(this.#options.pageSize));
|
|
154
287
|
if (gitea) {
|
|
@@ -156,6 +289,8 @@ export class ForgeInventoryAdapter {
|
|
|
156
289
|
if (state.collection === 'repositories') {
|
|
157
290
|
url.searchParams.set('sort', 'id'); url.searchParams.set('order', 'asc'); url.searchParams.set('private', 'true');
|
|
158
291
|
}
|
|
292
|
+
} else if (state.collection === 'namespaceDirectory') {
|
|
293
|
+
url.searchParams.set('page', String(state.page));
|
|
159
294
|
} else {
|
|
160
295
|
url.searchParams.set('order_by', 'id'); url.searchParams.set('sort', 'asc');
|
|
161
296
|
if (state.collection === 'repositories') {
|
|
@@ -173,8 +308,9 @@ export class ForgeInventoryAdapter {
|
|
|
173
308
|
return `${payload}.${plugins.crypto.createHmac('sha256', this.#cursorKey).update(payload).digest('base64url')}`;
|
|
174
309
|
}
|
|
175
310
|
|
|
176
|
-
private continuation(collection: TCollection, cursor?: string): IContinuation {
|
|
177
|
-
|
|
311
|
+
private continuation(collection: TCollection, cursor?: string, repositoryKey: string | null = null): IContinuation {
|
|
312
|
+
const directory = collection === 'namespaceDirectory' && this.#options.kind === 'gitea';
|
|
313
|
+
if (cursor === undefined) return { version: 3, collection, repositoryKey, page: 1, pagesRead: 0, phase: directory ? 'organizations' : null, after: null };
|
|
178
314
|
try {
|
|
179
315
|
if (cursor.length > 2048) throw new Error();
|
|
180
316
|
const [payload, signature, extra] = cursor.split('.');
|
|
@@ -182,18 +318,37 @@ export class ForgeInventoryAdapter {
|
|
|
182
318
|
if (extra !== undefined || !signature || signature.length !== expected.length ||
|
|
183
319
|
!plugins.crypto.timingSafeEqual(Buffer.from(signature), Buffer.from(expected))) throw new Error();
|
|
184
320
|
const state = JSON.parse(Buffer.from(payload, 'base64url').toString()) as IContinuation;
|
|
185
|
-
if (state.version !==
|
|
186
|
-
state.
|
|
321
|
+
if (state.version !== 3 || state.collection !== collection || state.repositoryKey !== repositoryKey || !Number.isSafeInteger(state.page) ||
|
|
322
|
+
!Number.isSafeInteger(state.pagesRead) || state.pagesRead < 1 || state.pagesRead >= this.#options.maxPages ||
|
|
323
|
+
state.page < 1 || state.page > state.pagesRead + 1 ||
|
|
324
|
+
(directory ? state.phase !== 'organizations' && state.phase !== 'users' : state.phase !== null) ||
|
|
325
|
+
((collection !== 'repositories' || this.#options.kind !== 'gitlab') && state.after !== null) ||
|
|
187
326
|
(state.after !== null && validation.decimal(state.after) !== state.after)) throw new Error();
|
|
188
327
|
return state;
|
|
189
328
|
} catch { throw new validation.ForgeInventoryError('invalid-cursor'); }
|
|
190
329
|
}
|
|
191
330
|
|
|
192
|
-
private async page<TItem>(collection: TCollection, map: (value: unknown) => TItem, requestArg: IForgeInventoryRequest): Promise<plugins.fossInterfaces.data.IForgeInventoryPage<TItem>> {
|
|
193
|
-
const
|
|
331
|
+
private async page<TItem>(collection: TCollection, map: (value: unknown, state: IContinuation) => TItem, requestArg: IForgeInventoryRequest, scope?: TIdentifiedRepositoryScope): Promise<plugins.fossInterfaces.data.IForgeInventoryPage<TItem>> {
|
|
332
|
+
const repositoryKey = scope ? plugins.crypto.createHash('sha256').update(JSON.stringify(scope)).digest('hex') : null;
|
|
333
|
+
const state = this.continuation(collection, requestArg.cursor, repositoryKey);
|
|
194
334
|
const startedAt = new Date().toISOString();
|
|
195
|
-
const url = this.url(state);
|
|
196
|
-
|
|
335
|
+
const url = this.url(state, scope);
|
|
336
|
+
let pageRequest = requestArg;
|
|
337
|
+
if (collection === 'issues' && this.#options.kind === 'gitea' && scope) {
|
|
338
|
+
// Gitea addresses issue lists by mutable path. Check ownership even for empty pages.
|
|
339
|
+
const before = this.#metrics.responseBytes;
|
|
340
|
+
const repositoryUrl = new URL(`${this.#options.baseUrl}/api/v1/repos/${scope.location.pathSegments.map(encodeURIComponent).join('/')}`);
|
|
341
|
+
const probe = await this.request(repositoryUrl, requestArg);
|
|
342
|
+
if (validation.decimal(validation.record(probe.body).id) !== scope.identity.ref.objectId) {
|
|
343
|
+
throw new validation.ForgeInventoryError('repository-identity-changed');
|
|
344
|
+
}
|
|
345
|
+
if (requestArg.maxResponseBytes !== undefined) {
|
|
346
|
+
const remaining = requestArg.maxResponseBytes - (this.#metrics.responseBytes - before);
|
|
347
|
+
if (remaining <= 0) throw new validation.ForgeInventoryError('response-size-limit');
|
|
348
|
+
pageRequest = { ...requestArg, maxResponseBytes: remaining };
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
const { body, headers } = await this.request(url, pageRequest);
|
|
197
352
|
let rows: unknown = body;
|
|
198
353
|
if (this.#options.kind === 'gitea' && collection === 'repositories') {
|
|
199
354
|
const envelope = validation.record(body);
|
|
@@ -201,17 +356,17 @@ export class ForgeInventoryAdapter {
|
|
|
201
356
|
rows = envelope.data;
|
|
202
357
|
}
|
|
203
358
|
if (!Array.isArray(rows) || rows.length > this.#options.pageSize) throw new validation.ForgeInventoryError('invalid-page');
|
|
204
|
-
const items = rows.map(map);
|
|
359
|
+
const items = rows.map((item) => map(item, state));
|
|
205
360
|
let pagination: plugins.fossInterfaces.data.TForgeInventoryPagination;
|
|
206
361
|
try {
|
|
207
362
|
const next = this.next(state, url, headers, rows);
|
|
208
|
-
pagination = next === null ? { status: 'exhausted' } : state.
|
|
363
|
+
pagination = next === null ? { status: 'exhausted' } : state.pagesRead + 1 >= this.#options.maxPages
|
|
209
364
|
? { status: 'incomplete', reason: 'Configured page limit reached.' }
|
|
210
365
|
: { status: 'more', cursor: this.cursor(next) };
|
|
211
366
|
} catch {
|
|
212
367
|
pagination = { status: 'incomplete', reason: 'Invalid or inconsistent pagination metadata.' };
|
|
213
368
|
}
|
|
214
|
-
return { schemaVersion: 1, instance: this.instance, scope: { kind: 'instance' }, observation: this.observation(startedAt), items, pagination };
|
|
369
|
+
return { schemaVersion: 1, instance: this.instance, scope: scope ?? { kind: 'instance' }, observation: this.observation(startedAt), items, pagination };
|
|
215
370
|
}
|
|
216
371
|
|
|
217
372
|
private next(state: IContinuation, current: URL, headers: Headers, rows: unknown[]): IContinuation | null {
|
|
@@ -247,19 +402,20 @@ export class ForgeInventoryAdapter {
|
|
|
247
402
|
if (!keyset && xNext !== null && xNext !== '' && validation.decimal(xNext) !== String(state.page + 1)) throw new Error();
|
|
248
403
|
if (rows.length === 0) {
|
|
249
404
|
if (nextUrl || xNext) throw new Error();
|
|
405
|
+
if (state.phase === 'organizations') return { ...state, phase: 'users', page: 1, pagesRead: state.pagesRead + 1 };
|
|
250
406
|
return null;
|
|
251
407
|
}
|
|
252
408
|
if (keyset) {
|
|
253
409
|
if (!nextUrl) return null; // GitLab keyset exhaustion is signalled by no next Link.
|
|
254
410
|
const after = validation.decimal(nextUrl.searchParams.get('id_after'));
|
|
255
411
|
if (after !== previousId.toString()) throw new Error();
|
|
256
|
-
return { ...state, page: state.page + 1, after };
|
|
412
|
+
return { ...state, page: state.page + 1, pagesRead: state.pagesRead + 1, after };
|
|
257
413
|
}
|
|
258
414
|
if (nextUrl && validation.decimal(nextUrl.searchParams.get('page')) !== String(state.page + 1)) throw new Error();
|
|
259
415
|
if (nextUrl && xNext === '') throw new Error();
|
|
260
416
|
// A short page can reflect a server page-size cap. Probe the following page
|
|
261
417
|
// even without Link rather than silently truncating at that cap.
|
|
262
|
-
return { ...state, page: state.page + 1 };
|
|
418
|
+
return { ...state, page: state.page + 1, pagesRead: state.pagesRead + 1 };
|
|
263
419
|
}
|
|
264
420
|
|
|
265
421
|
private async request(url: URL, requestArg: IForgeInventoryRequest): Promise<{ body: unknown; headers: Headers }> {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type * as plugins from './plugins.js';
|
|
2
2
|
import { ForgeInventoryAdapter, ForgeInventoryError } from './classes.forgeinventory.js';
|
|
3
|
-
import type { IForgeInventoryMetrics, IForgeInventoryRequest, IForgeInventoryOptions } from './classes.forgeinventory.js';
|
|
3
|
+
import type { IForgeInventoryMetrics, IForgeInventoryRequest, IForgeInventoryOptions, TForgeInventoryVersionInspection } from './classes.forgeinventory.js';
|
|
4
4
|
import { unknownSize } from './forgeinventory.validation.js';
|
|
5
5
|
|
|
6
6
|
export interface IForgeInventoryScanOptions {
|
|
@@ -9,15 +9,28 @@ export interface IForgeInventoryScanOptions {
|
|
|
9
9
|
maxItems?: number;
|
|
10
10
|
/** Deadline for the complete scan, including body reads. */
|
|
11
11
|
timeoutMs?: number;
|
|
12
|
-
/** Maximum response bytes consumed across
|
|
12
|
+
/** Maximum response bytes consumed across all collection and version requests. */
|
|
13
13
|
maxScanResponseBytes?: number;
|
|
14
|
+
/** Probe the server version at the beginning of this scan. Defaults to false. */
|
|
15
|
+
inspectVersion?: boolean;
|
|
16
|
+
/** Include personal namespaces through authenticated directory endpoints. Defaults to false. */
|
|
17
|
+
namespaceDirectory?: boolean;
|
|
18
|
+
/** Inspect all visible open/closed issues in the observed repositories. Defaults to false. */
|
|
19
|
+
includeIssues?: boolean;
|
|
14
20
|
}
|
|
15
21
|
|
|
16
22
|
export interface IForgeInventoryScan {
|
|
17
23
|
report: plugins.fossInterfaces.data.IForgeInventoryReport;
|
|
18
24
|
repositories: plugins.fossInterfaces.data.IForgeInventoryRepository[];
|
|
19
25
|
namespaces: plugins.fossInterfaces.data.IForgeInventoryNamespace[];
|
|
26
|
+
issues: plugins.fossInterfaces.data.IForgeInventoryIssue[];
|
|
20
27
|
metrics: IForgeInventoryMetrics;
|
|
28
|
+
versionInspection: TForgeInventoryVersionInspection | { status: 'notInspected'; reason: string };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
interface ICollectedItems<TItem> {
|
|
32
|
+
items: TItem[];
|
|
33
|
+
result: plugins.fossInterfaces.data.TForgeInventoryCollectionResult;
|
|
21
34
|
}
|
|
22
35
|
|
|
23
36
|
const entityClasses = {
|
|
@@ -48,27 +61,53 @@ export class ForgeInventoryScanner {
|
|
|
48
61
|
!Number.isSafeInteger(maxScanResponseBytes) || maxScanResponseBytes < 1 || maxScanResponseBytes > 1024 * 1024 * 1024) {
|
|
49
62
|
throw new ForgeInventoryError('invalid-scan-limits');
|
|
50
63
|
}
|
|
64
|
+
if ((optionsArg.inspectVersion !== undefined && typeof optionsArg.inspectVersion !== 'boolean') ||
|
|
65
|
+
(optionsArg.namespaceDirectory !== undefined && typeof optionsArg.namespaceDirectory !== 'boolean') ||
|
|
66
|
+
(optionsArg.includeIssues !== undefined && typeof optionsArg.includeIssues !== 'boolean')) {
|
|
67
|
+
throw new ForgeInventoryError('invalid-scan-options');
|
|
68
|
+
}
|
|
69
|
+
const { signal } = optionsArg;
|
|
70
|
+
const inspectVersion = optionsArg.inspectVersion ?? false;
|
|
71
|
+
const namespaceDirectory = optionsArg.namespaceDirectory ?? false;
|
|
72
|
+
const includeIssues = optionsArg.includeIssues ?? false;
|
|
51
73
|
if (this.#running) throw new ForgeInventoryError('scan-already-running');
|
|
52
74
|
this.#running = true;
|
|
53
75
|
const controller = new AbortController();
|
|
54
76
|
const abort = () => controller.abort();
|
|
55
|
-
if (
|
|
56
|
-
|
|
77
|
+
if (signal?.aborted) controller.abort();
|
|
78
|
+
signal?.addEventListener('abort', abort, { once: true });
|
|
57
79
|
const timeout = setTimeout(abort, timeoutMs);
|
|
58
80
|
const before = this.#adapter.metrics;
|
|
59
81
|
const deadline = Date.now() + timeoutMs;
|
|
60
82
|
const remainingBytes = () => maxScanResponseBytes - (this.#adapter.metrics.responseBytes - before.responseBytes);
|
|
61
83
|
const collections = Object.fromEntries(Object.keys(entityClasses).map((key) => [key, {
|
|
62
84
|
capability: { status: 'unverified', reason: 'No inventory reader has inspected this entity class.' },
|
|
63
|
-
result: { status: 'notInspected', reason: '
|
|
85
|
+
result: { status: 'notInspected', reason: 'No inventory reader has inspected this entity class in this scan.' },
|
|
64
86
|
}])) as plugins.fossInterfaces.data.IForgeInventoryReport['collections'];
|
|
65
87
|
try {
|
|
66
|
-
|
|
88
|
+
if (Date.now() >= deadline) controller.abort();
|
|
89
|
+
const versionInspection: IForgeInventoryScan['versionInspection'] = inspectVersion
|
|
90
|
+
? await this.#adapter.inspectVersion({ signal: controller.signal, maxResponseBytes: remainingBytes() })
|
|
91
|
+
: { status: 'notInspected', reason: 'Server version inspection was not requested.' };
|
|
92
|
+
const namespaces = await this.collect((request) => namespaceDirectory
|
|
93
|
+
? this.#adapter.listNamespaceDirectory(request) : this.#adapter.listNamespaces(request), maxItems, controller.signal, deadline, remainingBytes);
|
|
67
94
|
const repositories = await this.collect((request) => this.#adapter.listRepositories(request), maxItems, controller.signal, deadline, remainingBytes);
|
|
95
|
+
const issues = includeIssues
|
|
96
|
+
? await this.collectIssues(repositories, maxItems, controller.signal, deadline, remainingBytes) : null;
|
|
97
|
+
if (issues) collections.issues = {
|
|
98
|
+
capability: repositories.items.length === 0
|
|
99
|
+
? { status: 'unverified', reason: 'No repository issue endpoint was inspected.' }
|
|
100
|
+
: this.capability(issues, ['Only API-visible issue metadata is enumerated.', 'Comments, associations, attachments and historical import fidelity are not established.']),
|
|
101
|
+
result: issues.result,
|
|
102
|
+
};
|
|
68
103
|
const namespaceLimitation = 'Only organization/group namespaces were inspected; personal namespaces and full private namespace coverage remain uninspected.';
|
|
69
104
|
collections.namespaces = {
|
|
70
|
-
capability: this.capability(namespaces, [
|
|
71
|
-
|
|
105
|
+
capability: this.capability(namespaces, [
|
|
106
|
+
...(namespaceDirectory ? [] : [namespaceLimitation]),
|
|
107
|
+
'Live API visibility does not prove administrator completeness.',
|
|
108
|
+
'Namespace metadata does not establish user identity or permission inventory.',
|
|
109
|
+
]),
|
|
110
|
+
result: !namespaceDirectory && namespaces.result.status === 'enumerated' ? {
|
|
72
111
|
status: 'incomplete', observation: namespaces.result.observation,
|
|
73
112
|
observedCount: namespaces.result.count, reason: namespaceLimitation,
|
|
74
113
|
} : namespaces.result,
|
|
@@ -87,14 +126,16 @@ export class ForgeInventoryScanner {
|
|
|
87
126
|
const after = this.#adapter.metrics;
|
|
88
127
|
return {
|
|
89
128
|
report: {
|
|
90
|
-
schemaVersion: 1,
|
|
91
|
-
|
|
129
|
+
schemaVersion: 1,
|
|
130
|
+
instance: { ...this.#adapter.instance, version: versionInspection.status === 'observed' ? versionInspection.version : null },
|
|
131
|
+
scope: { kind: 'instance' }, collections,
|
|
132
|
+
additionalCollections: namespaceDirectory ? [] : [{
|
|
92
133
|
name: this.#adapter.instance.kind === 'gitea' ? 'gitea.organizations' : 'gitlab.groups',
|
|
93
134
|
capability: this.capability(namespaces, ['Endpoint visibility and live pagination limits apply.']),
|
|
94
135
|
result: namespaces.result,
|
|
95
136
|
}],
|
|
96
137
|
},
|
|
97
|
-
repositories: repositories.items, namespaces: namespaces.items,
|
|
138
|
+
repositories: repositories.items, namespaces: namespaces.items, issues: issues?.items ?? [], versionInspection,
|
|
98
139
|
metrics: {
|
|
99
140
|
requests: after.requests - before.requests,
|
|
100
141
|
responseBytes: after.responseBytes - before.responseBytes,
|
|
@@ -102,7 +143,7 @@ export class ForgeInventoryScanner {
|
|
|
102
143
|
},
|
|
103
144
|
};
|
|
104
145
|
} finally {
|
|
105
|
-
controller.abort(); clearTimeout(timeout);
|
|
146
|
+
controller.abort(); clearTimeout(timeout); signal?.removeEventListener('abort', abort);
|
|
106
147
|
this.#running = false;
|
|
107
148
|
}
|
|
108
149
|
}
|
|
@@ -118,10 +159,10 @@ export class ForgeInventoryScanner {
|
|
|
118
159
|
private async collect<TItem extends { identity: plugins.fossInterfaces.data.TForgeInventoryIdentity }>(
|
|
119
160
|
read: (request: IForgeInventoryRequest) => Promise<plugins.fossInterfaces.data.IForgeInventoryPage<TItem>>,
|
|
120
161
|
maxItems: number, signal: AbortSignal, deadline: number, remainingBytes: () => number,
|
|
121
|
-
|
|
162
|
+
identities = new Set<string>(),
|
|
163
|
+
): Promise<ICollectedItems<TItem>> {
|
|
122
164
|
const startedAt = new Date().toISOString();
|
|
123
165
|
const items: TItem[] = [];
|
|
124
|
-
const identities = new Set<string>();
|
|
125
166
|
let cursor: string | undefined;
|
|
126
167
|
const incomplete = (reason: string) => ({ items, result: {
|
|
127
168
|
status: 'incomplete' as const, observation: this.#adapter.observation(startedAt), observedCount: String(items.length), reason,
|
|
@@ -149,4 +190,30 @@ export class ForgeInventoryScanner {
|
|
|
149
190
|
cursor = page.pagination.cursor;
|
|
150
191
|
}
|
|
151
192
|
}
|
|
193
|
+
|
|
194
|
+
private async collectIssues(
|
|
195
|
+
repositories: ICollectedItems<plugins.fossInterfaces.data.IForgeInventoryRepository>,
|
|
196
|
+
maxItems: number, signal: AbortSignal, deadline: number, remainingBytes: () => number,
|
|
197
|
+
): Promise<ICollectedItems<plugins.fossInterfaces.data.IForgeInventoryIssue>> {
|
|
198
|
+
const startedAt = new Date().toISOString();
|
|
199
|
+
const items: plugins.fossInterfaces.data.IForgeInventoryIssue[] = [];
|
|
200
|
+
const identities = new Set<string>();
|
|
201
|
+
const incomplete = (reason: string): ICollectedItems<plugins.fossInterfaces.data.IForgeInventoryIssue> => ({
|
|
202
|
+
items, result: { status: 'incomplete', observation: this.#adapter.observation(startedAt), observedCount: String(items.length), reason },
|
|
203
|
+
});
|
|
204
|
+
for (const repository of repositories.items) {
|
|
205
|
+
const collection = await this.collect((request) => this.#adapter.listIssues({
|
|
206
|
+
kind: 'repository', identity: repository.identity, location: repository.location,
|
|
207
|
+
}, request), maxItems - items.length, signal, deadline, remainingBytes, identities);
|
|
208
|
+
for (const item of collection.items) items.push(item);
|
|
209
|
+
if (collection.result.status !== 'enumerated') {
|
|
210
|
+
return incomplete('reason' in collection.result ? collection.result.reason : 'Repository issue scan did not complete.');
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
if (signal.aborted || Date.now() >= deadline) return incomplete('Scan aborted or deadline reached.');
|
|
214
|
+
if (repositories.result.status !== 'enumerated') return incomplete('The repository inventory was incomplete.');
|
|
215
|
+
return {
|
|
216
|
+
items, result: { status: 'enumerated', observation: this.#adapter.observation(startedAt), count: String(items.length), sizeBytes: unknownSize('Issue storage bytes were not measured.') },
|
|
217
|
+
};
|
|
218
|
+
}
|
|
152
219
|
}
|
|
@@ -37,6 +37,19 @@ export const optionalBoolean = (value: unknown): boolean | null => {
|
|
|
37
37
|
return value;
|
|
38
38
|
};
|
|
39
39
|
|
|
40
|
+
/** Keep provider timestamp precision and offset; reject invalid calendar dates. */
|
|
41
|
+
export const timestamp = (value: unknown): string | null => {
|
|
42
|
+
if (value === undefined || value === null) return null;
|
|
43
|
+
if (typeof value !== 'string') throw new ForgeInventoryError('invalid-timestamp');
|
|
44
|
+
const match = value.match(/^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d{1,9})?(Z|[+-]\d{2}:\d{2})$/);
|
|
45
|
+
if (!match || !Number.isFinite(Date.parse(value)) || Number(match[2]) < 1 || Number(match[2]) > 12 ||
|
|
46
|
+
Number(match[3]) < 1 || Number(match[3]) > new Date(Date.UTC(Number(match[1]), Number(match[2]), 0)).getUTCDate() ||
|
|
47
|
+
Number(match[4]) > 23 || Number(match[5]) > 59 || Number(match[6]) > 59) {
|
|
48
|
+
throw new ForgeInventoryError('invalid-timestamp');
|
|
49
|
+
}
|
|
50
|
+
return value;
|
|
51
|
+
};
|
|
52
|
+
|
|
40
53
|
export const segments = (value: unknown): string[] => {
|
|
41
54
|
const path = optionalString(value);
|
|
42
55
|
if (!path || path.split('/').some((part) => !part || part === '.' || part === '..' || /[\\\u0000-\u001f\u007f]/.test(part))) {
|