@foss.global/codefeed 1.7.2 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{npmextra.json → .smartconfig.json} +15 -7
- package/dist_ts/00_commitinfo_data.js +3 -3
- package/dist_ts/classes.errors.d.ts +3 -0
- package/dist_ts/classes.errors.js +7 -0
- package/dist_ts/classes.gitmanagerprovider.d.ts +22 -0
- package/dist_ts/classes.gitmanagerprovider.js +129 -0
- package/dist_ts/index.d.ts +10 -13
- package/dist_ts/index.js +61 -25
- package/dist_ts/interfaces/index.d.ts +28 -11
- package/dist_ts/plugins.d.ts +5 -4
- package/dist_ts/plugins.js +3 -5
- package/license.md +21 -0
- package/package.json +12 -10
- package/readme.md +58 -5
- package/ts/00_commitinfo_data.ts +2 -2
- package/ts/classes.errors.ts +6 -0
- package/ts/classes.gitmanagerprovider.ts +157 -0
- package/ts/index.ts +73 -41
- package/ts/interfaces/index.ts +32 -11
- package/ts/plugins.ts +7 -7
- package/dist_ts/codefeed.plugins.d.ts +0 -7
- package/dist_ts/codefeed.plugins.js +0 -10
package/license.md
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) Task Venture Capital GmbH
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@foss.global/codefeed",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.9.0",
|
|
4
4
|
"private": false,
|
|
5
|
-
"description": "The @foss.global/codefeed module
|
|
5
|
+
"description": "The @foss.global/codefeed module generates commit/activity feeds from forge providers including Gitea and GitManager.",
|
|
6
6
|
"exports": {
|
|
7
7
|
".": "./dist_ts/index.js",
|
|
8
8
|
"./interfaces": "./dist_ts/interfaces/index.js"
|
|
@@ -11,14 +11,15 @@
|
|
|
11
11
|
"author": "Task Venture Capital GmbH",
|
|
12
12
|
"license": "MIT",
|
|
13
13
|
"devDependencies": {
|
|
14
|
-
"@git.zone/tsbuild": "^
|
|
15
|
-
"@git.zone/tsbundle": "^2.
|
|
16
|
-
"@git.zone/tsrun": "^
|
|
17
|
-
"@git.zone/tstest": "^
|
|
18
|
-
"@
|
|
19
|
-
"@types/node": "^22.15.2"
|
|
14
|
+
"@git.zone/tsbuild": "^4.4.2",
|
|
15
|
+
"@git.zone/tsbundle": "^2.11.0",
|
|
16
|
+
"@git.zone/tsrun": "^2.0.4",
|
|
17
|
+
"@git.zone/tstest": "^3.6.6",
|
|
18
|
+
"@types/node": "^26.0.0"
|
|
20
19
|
},
|
|
21
20
|
"dependencies": {
|
|
21
|
+
"@foss.global/gitmanager": "^0.3.0",
|
|
22
|
+
"@foss.global/interfaces": "^0.3.0",
|
|
22
23
|
"@push.rocks/lik": "^6.2.2",
|
|
23
24
|
"@push.rocks/qenv": "^6.1.3",
|
|
24
25
|
"@push.rocks/smartnpm": "^2.0.6",
|
|
@@ -42,8 +43,9 @@
|
|
|
42
43
|
"dist_ts_web/**/*",
|
|
43
44
|
"assets/**/*",
|
|
44
45
|
"cli.js",
|
|
45
|
-
"
|
|
46
|
-
"readme.md"
|
|
46
|
+
".smartconfig.json",
|
|
47
|
+
"readme.md",
|
|
48
|
+
"license.md"
|
|
47
49
|
],
|
|
48
50
|
"keywords": [
|
|
49
51
|
"codefeed",
|
package/readme.md
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
# @foss.global/codefeed
|
|
2
2
|
|
|
3
|
-
Generate an activity feed from a Gitea instance
|
|
3
|
+
Generate an activity feed from forge providers. The default provider scans a Gitea instance; `CodeFeedGitManagerProvider` adapts a local `@foss.global/gitmanager` instance. CodeFeed retrieves commits since a configurable timestamp and enriches them with tags, optional npm publish detection, and CHANGELOG snippets.
|
|
4
|
+
|
|
5
|
+
## Issue Reporting and Security
|
|
6
|
+
|
|
7
|
+
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
|
|
4
8
|
|
|
5
9
|
## Install
|
|
6
10
|
|
|
7
11
|
```bash
|
|
8
12
|
pnpm add @foss.global/codefeed
|
|
9
|
-
# or
|
|
10
|
-
npm i @foss.global/codefeed
|
|
11
13
|
```
|
|
12
14
|
|
|
13
15
|
Requires Node.js 18+ (global fetch/Request/Response) and ESM.
|
|
@@ -45,6 +47,31 @@ const feed = new CodeFeed('https://code.example.com', 'gitea_token', since, {
|
|
|
45
47
|
const commits = await feed.fetchAllCommitsFromInstance();
|
|
46
48
|
```
|
|
47
49
|
|
|
50
|
+
Advanced callers can provide a custom `ICodeFeedProvider` through the constructor options. The default provider keeps the existing Gitea HTTP behavior, while the provider contract lets adapters supply organizations, repositories, commits, tags, and changelog content from another backend.
|
|
51
|
+
|
|
52
|
+
### With GitManager
|
|
53
|
+
|
|
54
|
+
```ts
|
|
55
|
+
import { CodeFeed, CodeFeedGitManagerProvider } from '@foss.global/codefeed';
|
|
56
|
+
import { GitManager } from '@foss.global/gitmanager';
|
|
57
|
+
|
|
58
|
+
const since = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString();
|
|
59
|
+
const gitManager = new GitManager({
|
|
60
|
+
dataDir: '/var/lib/foss-global/gitmanager',
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
await gitManager.start();
|
|
64
|
+
|
|
65
|
+
const feed = new CodeFeed('gitmanager://local', undefined, since, {
|
|
66
|
+
provider: new CodeFeedGitManagerProvider({ gitManager }),
|
|
67
|
+
enableNpmCheck: false,
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
const commits = await feed.fetchAllCommitsFromInstance();
|
|
71
|
+
|
|
72
|
+
await gitManager.stop();
|
|
73
|
+
```
|
|
74
|
+
|
|
48
75
|
Each returned item follows this shape:
|
|
49
76
|
|
|
50
77
|
```ts
|
|
@@ -62,6 +89,8 @@ interface ICommitResult {
|
|
|
62
89
|
}
|
|
63
90
|
```
|
|
64
91
|
|
|
92
|
+
The canonical public result contract is exported as `ICodefeedCommitResult` from `@foss.global/interfaces`. `@foss.global/codefeed/interfaces` keeps `ICommitResult` as a package-local alias for that contract.
|
|
93
|
+
|
|
65
94
|
## Features
|
|
66
95
|
|
|
67
96
|
- Pagination for orgs, repos, commits, and tags (no missing pages)
|
|
@@ -69,6 +98,7 @@ interface ICommitResult {
|
|
|
69
98
|
- CHANGELOG discovery with case variants (`CHANGELOG.md`, `changelog.md`, `docs/CHANGELOG.md`)
|
|
70
99
|
- Tag-to-version mapping based on tag names (`vX.Y.Z` → `X.Y.Z`)
|
|
71
100
|
- Optional npm publish detection via `@org/repo` package versions
|
|
101
|
+
- GitManager provider adapter for repositories managed by `@foss.global/gitmanager`
|
|
72
102
|
- In-memory caching with window trimming and stable sorting
|
|
73
103
|
- Allow/deny filters for orgs and repos, optional time upper bound
|
|
74
104
|
- One-line metrics summary when `verbose: true`
|
|
@@ -76,13 +106,15 @@ interface ICommitResult {
|
|
|
76
106
|
## Environment
|
|
77
107
|
|
|
78
108
|
- Gitea base URL and an optional token with read access
|
|
109
|
+
- For GitManager-backed feeds, a started `GitManager` instance with imported or local repositories
|
|
79
110
|
- Node.js 18+ (global fetch)
|
|
80
111
|
|
|
81
112
|
## Testing
|
|
82
113
|
|
|
83
114
|
The repo contains:
|
|
84
|
-
- An integration test using a `GITEA_TOKEN` from `.nogit/` via `@push.rocks/qenv`.
|
|
115
|
+
- An opt-in integration test using a `GITEA_TOKEN` from `.nogit/` via `@push.rocks/qenv`.
|
|
85
116
|
- A mocked pagination test that does not require network.
|
|
117
|
+
- A GitManager provider mapping test using a local test stub.
|
|
86
118
|
|
|
87
119
|
Run tests:
|
|
88
120
|
|
|
@@ -90,10 +122,31 @@ Run tests:
|
|
|
90
122
|
pnpm test
|
|
91
123
|
```
|
|
92
124
|
|
|
93
|
-
For the integration test, ensure `GITEA_TOKEN` is provided
|
|
125
|
+
For the integration test, set `CODEFEED_RUN_LIVE=true` and ensure `GITEA_TOKEN` is provided, e.g. via `.nogit/` as used by `test/test.live.node.ts`.
|
|
94
126
|
|
|
95
127
|
## Notes
|
|
96
128
|
|
|
97
129
|
- When `taggedOnly` is enabled, the feed includes only commits associated with tags.
|
|
98
130
|
- `publishedOnNpm` is computed by matching the tag-derived version against the npm registry for `@org/repo`.
|
|
99
131
|
- For very large instances, consider using allowlists/denylists and enabling caching for incremental runs.
|
|
132
|
+
|
|
133
|
+
## License and Legal Information
|
|
134
|
+
|
|
135
|
+
This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the repository license file.
|
|
136
|
+
|
|
137
|
+
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
|
138
|
+
|
|
139
|
+
### Trademarks
|
|
140
|
+
|
|
141
|
+
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH or third parties, and are not included within the scope of the MIT license granted herein.
|
|
142
|
+
|
|
143
|
+
Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines or the guidelines of the respective third-party owners, and any usage must be approved in writing. Third-party trademarks used herein are the property of their respective owners and used only in a descriptive manner, e.g. for an implementation of an API or similar.
|
|
144
|
+
|
|
145
|
+
### Company Information
|
|
146
|
+
|
|
147
|
+
Task Venture Capital GmbH<br>
|
|
148
|
+
Registered at District Court Bremen HRB 35230 HB, Germany
|
|
149
|
+
|
|
150
|
+
For any legal inquiries or further information, please contact us via email at hello@task.vc.
|
|
151
|
+
|
|
152
|
+
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
package/ts/00_commitinfo_data.ts
CHANGED
|
@@ -3,6 +3,6 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export const commitinfo = {
|
|
5
5
|
name: '@foss.global/codefeed',
|
|
6
|
-
version: '1.
|
|
7
|
-
description: 'The @foss.global/codefeed module
|
|
6
|
+
version: '1.9.0',
|
|
7
|
+
description: 'The @foss.global/codefeed module generates commit/activity feeds from forge providers including Gitea and GitManager.'
|
|
8
8
|
}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import * as plugins from './plugins.js';
|
|
2
|
+
import { CodeFeedProviderCompletenessError } from './classes.errors.js';
|
|
3
|
+
|
|
4
|
+
export interface ICodeFeedGitManagerProviderOptions {
|
|
5
|
+
gitManager: plugins.TGitManager;
|
|
6
|
+
commitLimit?: number;
|
|
7
|
+
tagLimit?: number;
|
|
8
|
+
changelogPaths?: string[];
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
const defaultChangelogPaths = [
|
|
12
|
+
'CHANGELOG.md',
|
|
13
|
+
'changelog.md',
|
|
14
|
+
'Changelog.md',
|
|
15
|
+
'docs/CHANGELOG.md',
|
|
16
|
+
];
|
|
17
|
+
|
|
18
|
+
export class CodeFeedGitManagerProvider implements plugins.interfaces.ICodeFeedProvider {
|
|
19
|
+
private gitManager: plugins.TGitManager;
|
|
20
|
+
private commitLimit: number;
|
|
21
|
+
private tagLimit: number;
|
|
22
|
+
private changelogPaths: string[];
|
|
23
|
+
|
|
24
|
+
constructor(optionsArg: ICodeFeedGitManagerProviderOptions) {
|
|
25
|
+
this.gitManager = optionsArg.gitManager;
|
|
26
|
+
this.commitLimit = optionsArg.commitLimit ?? 1000;
|
|
27
|
+
this.tagLimit = optionsArg.tagLimit ?? 1000;
|
|
28
|
+
this.changelogPaths = optionsArg.changelogPaths ?? defaultChangelogPaths;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
public async listOrganizations(): Promise<string[]> {
|
|
32
|
+
const repositories = await this.gitManager.listRepositories();
|
|
33
|
+
return [...new Set(repositories.map((repository) => repository.org))]
|
|
34
|
+
.sort((a, b) => a.localeCompare(b));
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
public async listRepositoriesForOrg(orgArg: string): Promise<plugins.interfaces.IRepository[]> {
|
|
38
|
+
const repositories = await this.gitManager.listRepositories();
|
|
39
|
+
return repositories
|
|
40
|
+
.filter((repository) => repository.org === orgArg)
|
|
41
|
+
.map((repository) => ({
|
|
42
|
+
owner: {
|
|
43
|
+
login: repository.org,
|
|
44
|
+
},
|
|
45
|
+
name: repository.repo,
|
|
46
|
+
}))
|
|
47
|
+
.sort((a, b) => a.name.localeCompare(b.name));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
public async getLatestCommitForRepo(ownerArg: string, repoArg: string): Promise<plugins.interfaces.ICommit | undefined> {
|
|
51
|
+
const commits = await this.gitManager.listCommits({
|
|
52
|
+
org: ownerArg,
|
|
53
|
+
repo: repoArg,
|
|
54
|
+
limit: 1,
|
|
55
|
+
});
|
|
56
|
+
const [commit] = commits;
|
|
57
|
+
if (!commit) {
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
return this.mapCommitSummary(commit);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
public async listCommitsForRepo(
|
|
64
|
+
ownerArg: string,
|
|
65
|
+
repoArg: string,
|
|
66
|
+
sinceTimestampArg?: string
|
|
67
|
+
): Promise<plugins.interfaces.ICommit[]> {
|
|
68
|
+
const sinceTimestamp = this.parseSinceTimestamp(sinceTimestampArg);
|
|
69
|
+
const commits = await this.gitManager.listCommits({
|
|
70
|
+
org: ownerArg,
|
|
71
|
+
repo: repoArg,
|
|
72
|
+
limit: this.commitLimit,
|
|
73
|
+
});
|
|
74
|
+
if (sinceTimestamp !== undefined && commits.length >= this.commitLimit) {
|
|
75
|
+
throw new CodeFeedProviderCompletenessError(
|
|
76
|
+
`GitManager commit listing for ${ownerArg}/${repoArg} reached the configured limit of ${this.commitLimit}; complete since filtering requires GitManager pagination.`
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
const filteredCommits = sinceTimestamp === undefined
|
|
80
|
+
? commits
|
|
81
|
+
: commits.filter((commit) => Date.parse(commit.createdAt) > sinceTimestamp);
|
|
82
|
+
return Promise.all(filteredCommits.map((commit) => this.mapCommitSummary(commit)));
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
public async listTagsForRepo(ownerArg: string, repoArg: string): Promise<plugins.interfaces.ICodeFeedTagInfo> {
|
|
86
|
+
const taggedShas = new Set<string>();
|
|
87
|
+
const tagNameBySha = new Map<string, string>();
|
|
88
|
+
const tags = await this.gitManager.listTags({
|
|
89
|
+
org: ownerArg,
|
|
90
|
+
repo: repoArg,
|
|
91
|
+
limit: this.tagLimit,
|
|
92
|
+
});
|
|
93
|
+
if (tags.length >= this.tagLimit) {
|
|
94
|
+
throw new CodeFeedProviderCompletenessError(
|
|
95
|
+
`GitManager tag listing for ${ownerArg}/${repoArg} reached the configured limit of ${this.tagLimit}; complete tag enrichment requires GitManager pagination.`
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
for (const tag of tags) {
|
|
99
|
+
if (tag.targetObjectType !== 'commit') {
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
taggedShas.add(tag.targetObjectId);
|
|
103
|
+
tagNameBySha.set(tag.targetObjectId, tag.name);
|
|
104
|
+
}
|
|
105
|
+
return {
|
|
106
|
+
shas: taggedShas,
|
|
107
|
+
map: tagNameBySha,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
public async readChangelogForRepo(ownerArg: string, repoArg: string): Promise<string> {
|
|
112
|
+
for (const changelogPath of this.changelogPaths) {
|
|
113
|
+
try {
|
|
114
|
+
const file = await this.gitManager.readFile({
|
|
115
|
+
org: ownerArg,
|
|
116
|
+
repo: repoArg,
|
|
117
|
+
path: changelogPath,
|
|
118
|
+
});
|
|
119
|
+
if (file.isBinary) {
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
return Buffer.from(file.contentBase64, 'base64').toString('utf8');
|
|
123
|
+
} catch {
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return '';
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
private async mapCommitSummary(commitArg: plugins.TGitManagerCommitSummary): Promise<plugins.interfaces.ICommit> {
|
|
131
|
+
const commitDetail = await this.gitManager.getCommit({
|
|
132
|
+
org: commitArg.org,
|
|
133
|
+
repo: commitArg.repo,
|
|
134
|
+
ref: commitArg.sha,
|
|
135
|
+
});
|
|
136
|
+
return {
|
|
137
|
+
sha: commitDetail.sha,
|
|
138
|
+
commit: {
|
|
139
|
+
message: commitDetail.message,
|
|
140
|
+
author: {
|
|
141
|
+
date: commitDetail.author.date,
|
|
142
|
+
},
|
|
143
|
+
},
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
private parseSinceTimestamp(sinceTimestampArg?: string): number | undefined {
|
|
148
|
+
if (!sinceTimestampArg) {
|
|
149
|
+
return undefined;
|
|
150
|
+
}
|
|
151
|
+
const timestamp = Date.parse(sinceTimestampArg);
|
|
152
|
+
if (Number.isNaN(timestamp)) {
|
|
153
|
+
throw new Error(`Invalid since timestamp: ${sinceTimestampArg}`);
|
|
154
|
+
}
|
|
155
|
+
return timestamp;
|
|
156
|
+
}
|
|
157
|
+
}
|
package/ts/index.ts
CHANGED
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
|
+
import { CodeFeedProviderCompletenessError } from './classes.errors.js';
|
|
3
|
+
|
|
4
|
+
export { CodeFeedProviderCompletenessError } from './classes.errors.js';
|
|
5
|
+
export { CodeFeedGitManagerProvider } from './classes.gitmanagerprovider.js';
|
|
6
|
+
export type { ICodeFeedGitManagerProviderOptions } from './classes.gitmanagerprovider.js';
|
|
2
7
|
|
|
3
8
|
export class CodeFeed {
|
|
4
9
|
private baseUrl: string;
|
|
@@ -7,12 +12,12 @@ export class CodeFeed {
|
|
|
7
12
|
private pageLimit = 50;
|
|
8
13
|
// Raw changelog content for the current repository
|
|
9
14
|
private changelogContent: string = '';
|
|
10
|
-
|
|
11
|
-
private npmRegistry: plugins.smartnpm.NpmRegistry;
|
|
15
|
+
private npmRegistry?: plugins.TNpmRegistry;
|
|
12
16
|
// In-memory stateful cache of commits
|
|
13
17
|
private enableCache: boolean = false;
|
|
14
18
|
private cacheWindowMs?: number;
|
|
15
19
|
private cache: plugins.interfaces.ICommitResult[] = [];
|
|
20
|
+
private provider: plugins.interfaces.ICodeFeedProvider;
|
|
16
21
|
// enable or disable npm publishedOnNpm checks (true by default)
|
|
17
22
|
private enableNpmCheck: boolean = true;
|
|
18
23
|
// return only tagged commits (false by default)
|
|
@@ -29,18 +34,7 @@ export class CodeFeed {
|
|
|
29
34
|
baseUrl: string,
|
|
30
35
|
token?: string,
|
|
31
36
|
lastRunTimestamp?: string,
|
|
32
|
-
options?:
|
|
33
|
-
enableCache?: boolean;
|
|
34
|
-
cacheWindowMs?: number;
|
|
35
|
-
enableNpmCheck?: boolean;
|
|
36
|
-
taggedOnly?: boolean;
|
|
37
|
-
orgAllowlist?: string[];
|
|
38
|
-
orgDenylist?: string[];
|
|
39
|
-
repoAllowlist?: string[];
|
|
40
|
-
repoDenylist?: string[];
|
|
41
|
-
untilTimestamp?: string;
|
|
42
|
-
verbose?: boolean;
|
|
43
|
-
}
|
|
37
|
+
options?: plugins.interfaces.ICodeFeedOptions
|
|
44
38
|
) {
|
|
45
39
|
this.baseUrl = baseUrl;
|
|
46
40
|
this.token = token;
|
|
@@ -58,8 +52,7 @@ export class CodeFeed {
|
|
|
58
52
|
this.untilTimestamp = options?.untilTimestamp;
|
|
59
53
|
this.verbose = options?.verbose ?? false;
|
|
60
54
|
this.cache = [];
|
|
61
|
-
|
|
62
|
-
this.npmRegistry = new plugins.smartnpm.NpmRegistry();
|
|
55
|
+
this.provider = options?.provider ?? this.createDefaultProvider();
|
|
63
56
|
console.log('CodeFeed initialized with last run timestamp:', this.lastRunTimestamp);
|
|
64
57
|
}
|
|
65
58
|
|
|
@@ -81,7 +74,7 @@ export class CodeFeed {
|
|
|
81
74
|
}
|
|
82
75
|
|
|
83
76
|
// 1) get all organizations
|
|
84
|
-
let orgs = await this.
|
|
77
|
+
let orgs = await this.provider.listOrganizations();
|
|
85
78
|
// apply allow/deny filters
|
|
86
79
|
if (this.orgAllowlist && this.orgAllowlist.length > 0) {
|
|
87
80
|
orgs = orgs.filter((o) => this.orgAllowlist!.includes(o));
|
|
@@ -93,7 +86,7 @@ export class CodeFeed {
|
|
|
93
86
|
// 2) fetch repos per org in parallel
|
|
94
87
|
const repoLists = await Promise.all(
|
|
95
88
|
orgs.map((org) =>
|
|
96
|
-
stack.getNonExclusiveExecutionSlot(() => this.
|
|
89
|
+
stack.getNonExclusiveExecutionSlot(() => this.provider.listRepositoriesForOrg(org))
|
|
97
90
|
)
|
|
98
91
|
);
|
|
99
92
|
// flatten to [{ owner, name }]
|
|
@@ -114,31 +107,25 @@ export class CodeFeed {
|
|
|
114
107
|
const commitJobs = allRepos.map(({ owner, name }) =>
|
|
115
108
|
stack.getNonExclusiveExecutionSlot(async () => {
|
|
116
109
|
try {
|
|
117
|
-
|
|
118
|
-
const probeResp = await this.fetchFunction(
|
|
119
|
-
`/api/v1/repos/${owner}/${name}/commits?limit=1`,
|
|
120
|
-
{ headers: this.token ? { Authorization: `token ${this.token}` } : {} }
|
|
121
|
-
);
|
|
122
|
-
if (!probeResp.ok) {
|
|
123
|
-
throw new Error(`Probe failed for ${owner}/${name}: ${probeResp.statusText}`);
|
|
124
|
-
}
|
|
125
|
-
const probeData: plugins.interfaces.ICommit[] = await probeResp.json();
|
|
110
|
+
const latestCommit = await this.provider.getLatestCommitForRepo(owner, name);
|
|
126
111
|
// If no commits or no new commits since last run, skip
|
|
127
112
|
if (
|
|
128
|
-
|
|
129
|
-
new Date(
|
|
130
|
-
new Date(effectiveSince).getTime()
|
|
113
|
+
!latestCommit ||
|
|
114
|
+
new Date(latestCommit.commit.author.date).getTime() <= new Date(effectiveSince).getTime()
|
|
131
115
|
) {
|
|
132
116
|
return { owner, name, commits: [] };
|
|
133
117
|
}
|
|
134
118
|
// 3b) Fetch commits since last run
|
|
135
|
-
const commits = await this.
|
|
119
|
+
const commits = await this.provider.listCommitsForRepo(
|
|
136
120
|
owner,
|
|
137
121
|
name,
|
|
138
122
|
effectiveSince
|
|
139
123
|
);
|
|
140
124
|
return { owner, name, commits };
|
|
141
125
|
} catch (e: any) {
|
|
126
|
+
if (e instanceof CodeFeedProviderCompletenessError) {
|
|
127
|
+
throw e;
|
|
128
|
+
}
|
|
142
129
|
console.error(`Failed to fetch commits for ${owner}/${name}:`, e.message);
|
|
143
130
|
return { owner, name, commits: [] };
|
|
144
131
|
}
|
|
@@ -157,15 +144,18 @@ export class CodeFeed {
|
|
|
157
144
|
}
|
|
158
145
|
reposWithNewCommits++;
|
|
159
146
|
// load changelog for this repo
|
|
160
|
-
await this.
|
|
147
|
+
this.changelogContent = await this.provider.readChangelogForRepo(owner, name);
|
|
161
148
|
// fetch tags for this repo
|
|
162
149
|
let taggedShas: Set<string>;
|
|
163
150
|
let tagNameBySha: Map<string, string>;
|
|
164
151
|
try {
|
|
165
|
-
const tagInfo = await this.
|
|
152
|
+
const tagInfo = await this.provider.listTagsForRepo(owner, name);
|
|
166
153
|
taggedShas = tagInfo.shas;
|
|
167
154
|
tagNameBySha = tagInfo.map;
|
|
168
155
|
} catch (e: any) {
|
|
156
|
+
if (e instanceof CodeFeedProviderCompletenessError) {
|
|
157
|
+
throw e;
|
|
158
|
+
}
|
|
169
159
|
console.error(`Failed to fetch tags for ${owner}/${name}:`, e.message);
|
|
170
160
|
taggedShas = new Set<string>();
|
|
171
161
|
tagNameBySha = new Map<string, string>();
|
|
@@ -175,7 +165,8 @@ export class CodeFeed {
|
|
|
175
165
|
let pkgInfo: { allVersions: Array<{ version: string }> } | null = null;
|
|
176
166
|
if (hasTaggedCommit && this.enableNpmCheck) {
|
|
177
167
|
try {
|
|
178
|
-
|
|
168
|
+
const npmRegistry = await this.getNpmRegistry();
|
|
169
|
+
pkgInfo = await npmRegistry.getPackageInfo(`@${owner}/${name}`);
|
|
179
170
|
} catch (e: any) {
|
|
180
171
|
console.error(`Failed to fetch package info for ${owner}/${name}:`, e.message);
|
|
181
172
|
pkgInfo = null;
|
|
@@ -270,9 +261,38 @@ export class CodeFeed {
|
|
|
270
261
|
/**
|
|
271
262
|
* Load the changelog directly from the Gitea repository.
|
|
272
263
|
*/
|
|
273
|
-
private
|
|
264
|
+
private createDefaultProvider(): plugins.interfaces.ICodeFeedProvider {
|
|
265
|
+
return {
|
|
266
|
+
listOrganizations: () => this.fetchAllOrganizations(),
|
|
267
|
+
listRepositoriesForOrg: (org) => this.fetchRepositoriesForOrg(org),
|
|
268
|
+
getLatestCommitForRepo: (owner, repo) => this.fetchLatestCommitForRepo(owner, repo),
|
|
269
|
+
listCommitsForRepo: (owner, repo, sinceTimestamp) => this.fetchRecentCommitsForRepo(owner, repo, sinceTimestamp),
|
|
270
|
+
listTagsForRepo: (owner, repo) => this.fetchTags(owner, repo),
|
|
271
|
+
readChangelogForRepo: async (owner, repo) => {
|
|
272
|
+
await this.loadChangelogFromRepo(owner, repo);
|
|
273
|
+
return this.changelogContent;
|
|
274
|
+
},
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
private createAuthHeaders(): Record<string, string> {
|
|
274
279
|
const headers: Record<string, string> = {};
|
|
275
|
-
if (this.token)
|
|
280
|
+
if (this.token) {
|
|
281
|
+
headers.Authorization = `token ${this.token}`;
|
|
282
|
+
}
|
|
283
|
+
return headers;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
private async getNpmRegistry(): Promise<plugins.TNpmRegistry> {
|
|
287
|
+
if (!this.npmRegistry) {
|
|
288
|
+
const smartnpm = await plugins.getSmartnpm();
|
|
289
|
+
this.npmRegistry = new smartnpm.NpmRegistry();
|
|
290
|
+
}
|
|
291
|
+
return this.npmRegistry;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
private async loadChangelogFromRepo(owner: string, repo: string): Promise<void> {
|
|
295
|
+
const headers = this.createAuthHeaders();
|
|
276
296
|
const candidates = [
|
|
277
297
|
'CHANGELOG.md',
|
|
278
298
|
'changelog.md',
|
|
@@ -332,14 +352,14 @@ export class CodeFeed {
|
|
|
332
352
|
/**
|
|
333
353
|
* Fetch all tags for a given repo and return the set of tagged commit SHAs
|
|
334
354
|
*/
|
|
335
|
-
private async fetchTags(owner: string, repo: string): Promise<
|
|
355
|
+
private async fetchTags(owner: string, repo: string): Promise<plugins.interfaces.ICodeFeedTagInfo> {
|
|
336
356
|
const taggedShas = new Set<string>();
|
|
337
357
|
const tagNameBySha = new Map<string, string>();
|
|
338
358
|
let page = 1;
|
|
339
359
|
while (true) {
|
|
340
360
|
const url = `/api/v1/repos/${owner}/${repo}/tags?limit=${this.pageLimit}&page=${page}`;
|
|
341
361
|
const resp = await this.fetchFunction(url, {
|
|
342
|
-
headers: this.
|
|
362
|
+
headers: this.createAuthHeaders(),
|
|
343
363
|
});
|
|
344
364
|
if (!resp.ok) {
|
|
345
365
|
console.error(`Failed to fetch tags for ${owner}/${repo}: ${resp.status} ${resp.statusText}`);
|
|
@@ -360,8 +380,20 @@ export class CodeFeed {
|
|
|
360
380
|
return { shas: taggedShas, map: tagNameBySha };
|
|
361
381
|
}
|
|
362
382
|
|
|
383
|
+
private async fetchLatestCommitForRepo(owner: string, repo: string): Promise<plugins.interfaces.ICommit | undefined> {
|
|
384
|
+
const resp = await this.fetchFunction(
|
|
385
|
+
`/api/v1/repos/${owner}/${repo}/commits?limit=1`,
|
|
386
|
+
{ headers: this.createAuthHeaders() }
|
|
387
|
+
);
|
|
388
|
+
if (!resp.ok) {
|
|
389
|
+
throw new Error(`Probe failed for ${owner}/${repo}: ${resp.statusText}`);
|
|
390
|
+
}
|
|
391
|
+
const data: plugins.interfaces.ICommit[] = await resp.json();
|
|
392
|
+
return data[0];
|
|
393
|
+
}
|
|
394
|
+
|
|
363
395
|
private async fetchAllOrganizations(): Promise<string[]> {
|
|
364
|
-
const headers = this.
|
|
396
|
+
const headers = this.createAuthHeaders();
|
|
365
397
|
let page = 1;
|
|
366
398
|
const orgs: string[] = [];
|
|
367
399
|
while (true) {
|
|
@@ -379,7 +411,7 @@ export class CodeFeed {
|
|
|
379
411
|
}
|
|
380
412
|
|
|
381
413
|
private async fetchRepositoriesForOrg(org: string): Promise<plugins.interfaces.IRepository[]> {
|
|
382
|
-
const headers = this.
|
|
414
|
+
const headers = this.createAuthHeaders();
|
|
383
415
|
let page = 1;
|
|
384
416
|
const repos: plugins.interfaces.IRepository[] = [];
|
|
385
417
|
while (true) {
|
|
@@ -402,7 +434,7 @@ export class CodeFeed {
|
|
|
402
434
|
sinceTimestamp?: string
|
|
403
435
|
): Promise<plugins.interfaces.ICommit[]> {
|
|
404
436
|
const since = sinceTimestamp ?? this.lastRunTimestamp;
|
|
405
|
-
const headers = this.
|
|
437
|
+
const headers = this.createAuthHeaders();
|
|
406
438
|
let page = 1;
|
|
407
439
|
const commits: plugins.interfaces.ICommit[] = [];
|
|
408
440
|
while (true) {
|
package/ts/interfaces/index.ts
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import type { ICodefeedCommitResult } from '@foss.global/interfaces';
|
|
2
|
+
|
|
3
|
+
export type { ICodefeedCommitResult } from '@foss.global/interfaces';
|
|
4
|
+
|
|
1
5
|
export interface IRepositoryOwner {
|
|
2
6
|
login: string;
|
|
3
7
|
}
|
|
@@ -32,15 +36,32 @@ export interface IRepoSearchResponse {
|
|
|
32
36
|
data: IRepository[];
|
|
33
37
|
}
|
|
34
38
|
|
|
35
|
-
export interface ICommitResult {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
39
|
+
export interface ICommitResult extends ICodefeedCommitResult {}
|
|
40
|
+
|
|
41
|
+
export interface ICodeFeedTagInfo {
|
|
42
|
+
shas: Set<string>;
|
|
43
|
+
map: Map<string, string>;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface ICodeFeedProvider {
|
|
47
|
+
listOrganizations(): Promise<string[]>;
|
|
48
|
+
listRepositoriesForOrg(org: string): Promise<IRepository[]>;
|
|
49
|
+
getLatestCommitForRepo(owner: string, repo: string): Promise<ICommit | undefined>;
|
|
50
|
+
listCommitsForRepo(owner: string, repo: string, sinceTimestamp?: string): Promise<ICommit[]>;
|
|
51
|
+
listTagsForRepo(owner: string, repo: string): Promise<ICodeFeedTagInfo>;
|
|
52
|
+
readChangelogForRepo(owner: string, repo: string): Promise<string>;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface ICodeFeedOptions {
|
|
56
|
+
enableCache?: boolean;
|
|
57
|
+
cacheWindowMs?: number;
|
|
58
|
+
enableNpmCheck?: boolean;
|
|
59
|
+
taggedOnly?: boolean;
|
|
60
|
+
orgAllowlist?: string[];
|
|
61
|
+
orgDenylist?: string[];
|
|
62
|
+
repoAllowlist?: string[];
|
|
63
|
+
repoDenylist?: string[];
|
|
64
|
+
untilTimestamp?: string;
|
|
65
|
+
verbose?: boolean;
|
|
66
|
+
provider?: ICodeFeedProvider;
|
|
46
67
|
}
|
package/ts/plugins.ts
CHANGED
|
@@ -6,16 +6,16 @@ export {
|
|
|
6
6
|
}
|
|
7
7
|
|
|
8
8
|
// @push.rocks
|
|
9
|
-
import * as qenv from '@push.rocks/qenv';
|
|
10
|
-
import * as smartnpm from '@push.rocks/smartnpm';
|
|
11
|
-
import * as smartxml from '@push.rocks/smartxml';
|
|
12
9
|
import * as smarttime from '@push.rocks/smarttime';
|
|
13
10
|
import * as lik from '@push.rocks/lik';
|
|
14
11
|
|
|
12
|
+
export type TNpmRegistry = import('@push.rocks/smartnpm').NpmRegistry;
|
|
13
|
+
export type TGitManager = import('@foss.global/gitmanager').GitManager;
|
|
14
|
+
export type TGitManagerCommitSummary = import('@foss.global/gitmanager').IGitManagerCommitSummary;
|
|
15
|
+
|
|
16
|
+
export const getSmartnpm = async () => import('@push.rocks/smartnpm');
|
|
17
|
+
|
|
15
18
|
export {
|
|
16
|
-
qenv,
|
|
17
|
-
smartnpm,
|
|
18
|
-
smartxml,
|
|
19
19
|
smarttime,
|
|
20
20
|
lik,
|
|
21
|
-
}
|
|
21
|
+
}
|