@aligndottech/connector-core 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fetchers/confluence.d.ts +2 -1
- package/dist/fetchers/confluence.d.ts.map +1 -1
- package/dist/fetchers/confluence.js +9 -1
- package/dist/fetchers/confluence.js.map +1 -1
- package/dist/fetchers/git.d.ts +2 -1
- package/dist/fetchers/git.d.ts.map +1 -1
- package/dist/fetchers/git.js +8 -1
- package/dist/fetchers/git.js.map +1 -1
- package/dist/fetchers/github.d.ts +17 -4
- package/dist/fetchers/github.d.ts.map +1 -1
- package/dist/fetchers/github.js +152 -28
- package/dist/fetchers/github.js.map +1 -1
- package/dist/fetchers/gitlab.d.ts +4 -2
- package/dist/fetchers/gitlab.d.ts.map +1 -1
- package/dist/fetchers/gitlab.js +33 -4
- package/dist/fetchers/gitlab.js.map +1 -1
- package/dist/fetchers/jira.d.ts +2 -1
- package/dist/fetchers/jira.d.ts.map +1 -1
- package/dist/fetchers/jira.js +10 -2
- package/dist/fetchers/jira.js.map +1 -1
- package/dist/fetchers/linear.d.ts +2 -1
- package/dist/fetchers/linear.d.ts.map +1 -1
- package/dist/fetchers/linear.js +10 -2
- package/dist/fetchers/linear.js.map +1 -1
- package/dist/fetchers/notion.d.ts +4 -1
- package/dist/fetchers/notion.d.ts.map +1 -1
- package/dist/fetchers/notion.js +61 -30
- package/dist/fetchers/notion.js.map +1 -1
- package/dist/fetchers/slack.d.ts +12 -3
- package/dist/fetchers/slack.d.ts.map +1 -1
- package/dist/fetchers/slack.js +222 -28
- package/dist/fetchers/slack.js.map +1 -1
- package/dist/fetchers/teams.d.ts +2 -1
- package/dist/fetchers/teams.d.ts.map +1 -1
- package/dist/fetchers/teams.js +9 -1
- package/dist/fetchers/teams.js.map +1 -1
- package/dist/fetchers/util/time.d.ts +15 -0
- package/dist/fetchers/util/time.d.ts.map +1 -0
- package/dist/fetchers/util/time.js +24 -0
- package/dist/fetchers/util/time.js.map +1 -0
- package/dist/fetchers/zoom.d.ts +6 -1
- package/dist/fetchers/zoom.d.ts.map +1 -1
- package/dist/fetchers/zoom.js +102 -31
- package/dist/fetchers/zoom.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/types/fetcher.d.ts +54 -0
- package/dist/types/fetcher.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ConnectorFetcher, ConnectorFetcherOptions, FetcherItem } from '../types/fetcher.js';
|
|
1
|
+
import type { ConnectorFetcher, ConnectorFetcherOptions, FetcherItem, FetchResult } from '../types/fetcher.js';
|
|
2
2
|
/**
|
|
3
3
|
* Read-only personal Confluence fetcher (API v2): pages the token can read.
|
|
4
4
|
* OAuth (cloudId) or basic-auth (domain+email). Author = the page author.
|
|
@@ -6,5 +6,6 @@ import type { ConnectorFetcher, ConnectorFetcherOptions, FetcherItem } from '../
|
|
|
6
6
|
*/
|
|
7
7
|
export declare class ConfluenceFetcher implements ConnectorFetcher {
|
|
8
8
|
fetch(opts: ConnectorFetcherOptions): Promise<FetcherItem[]>;
|
|
9
|
+
fetchWithReport(opts: ConnectorFetcherOptions): Promise<FetchResult>;
|
|
9
10
|
}
|
|
10
11
|
//# sourceMappingURL=confluence.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"confluence.d.ts","sourceRoot":"","sources":["../../src/fetchers/confluence.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"confluence.d.ts","sourceRoot":"","sources":["../../src/fetchers/confluence.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAyD/G;;;;GAIG;AACH,qBAAa,iBAAkB,YAAW,gBAAgB;IAClD,KAAK,CAAC,IAAI,EAAE,uBAAuB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAI5D,eAAe,CAAC,IAAI,EAAE,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;CA0E3E"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { fetch } from 'undici';
|
|
2
2
|
import { FetcherAuthError } from './errors.js';
|
|
3
|
+
import { toIsoOrUndefined } from './util/time.js';
|
|
3
4
|
// Confluence v2 caps page size at 250 and paginates via _links.next (a cursor).
|
|
4
5
|
const CONFLUENCE_PAGE_MAX = 250;
|
|
5
6
|
function stripHtml(html) {
|
|
@@ -50,6 +51,9 @@ function makeConfluenceUserResolver(base, headers) {
|
|
|
50
51
|
*/
|
|
51
52
|
export class ConfluenceFetcher {
|
|
52
53
|
async fetch(opts) {
|
|
54
|
+
return (await this.fetchWithReport(opts)).items;
|
|
55
|
+
}
|
|
56
|
+
async fetchWithReport(opts) {
|
|
53
57
|
const cloudId = opts.cloudId;
|
|
54
58
|
const siteBase = opts.siteBase;
|
|
55
59
|
const email = opts.email;
|
|
@@ -70,6 +74,7 @@ export class ConfluenceFetcher {
|
|
|
70
74
|
: `https://${domain}`;
|
|
71
75
|
const resolveUser = makeConfluenceUserResolver(base, headers);
|
|
72
76
|
const items = [];
|
|
77
|
+
let scanned = 0;
|
|
73
78
|
let cursor;
|
|
74
79
|
let linkBase;
|
|
75
80
|
while (items.length < limit) {
|
|
@@ -91,16 +96,19 @@ export class ConfluenceFetcher {
|
|
|
91
96
|
for (const page of data.results ?? []) {
|
|
92
97
|
if (items.length >= limit)
|
|
93
98
|
break;
|
|
99
|
+
scanned += 1;
|
|
94
100
|
const bodyHtml = page.body?.storage?.value ?? '';
|
|
95
101
|
const bodyText = stripHtml(bodyHtml).slice(0, 2000);
|
|
96
102
|
const webui = page._links?.webui ?? '';
|
|
97
103
|
const pageUrl = webui.startsWith('http') ? webui : `${linkBase}${webui}`;
|
|
98
104
|
const author = await resolveUser(page.authorId);
|
|
105
|
+
const createdAt = toIsoOrUndefined(page.version?.createdAt);
|
|
99
106
|
items.push({
|
|
100
107
|
source_url: pageUrl,
|
|
101
108
|
platform: 'confluence',
|
|
102
109
|
raw_text: [page.title, bodyText].filter(Boolean).join('\n\n'),
|
|
103
110
|
title: page.title,
|
|
111
|
+
...(createdAt ? { created_at: createdAt } : {}),
|
|
104
112
|
...(author ? { author } : {}),
|
|
105
113
|
});
|
|
106
114
|
}
|
|
@@ -108,7 +116,7 @@ export class ConfluenceFetcher {
|
|
|
108
116
|
if (!cursor)
|
|
109
117
|
break;
|
|
110
118
|
}
|
|
111
|
-
return items;
|
|
119
|
+
return { items, report: { platform: 'confluence', scanned, requested: limit, skips: [] } };
|
|
112
120
|
}
|
|
113
121
|
}
|
|
114
122
|
//# sourceMappingURL=confluence.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"confluence.js","sourceRoot":"","sources":["../../src/fetchers/confluence.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAE/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"confluence.js","sourceRoot":"","sources":["../../src/fetchers/confluence.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAE/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAElD,gFAAgF;AAChF,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEhC,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACnE,CAAC;AAaD,oFAAoF;AACpF,SAAS,cAAc,CAAC,IAAwB;IAC9C,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,6BAA6B,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC;IAC9F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;mEACmE;AACnE,SAAS,0BAA0B,CAAC,IAAY,EAAE,OAA+B;IAC/E,MAAM,KAAK,GAAG,IAAI,GAAG,EAAmD,CAAC;IACzE,OAAO,KAAK,EAAE,SAAkB,EAAyD,EAAE;QACzF,IAAI,CAAC,SAAS;YAAE,OAAO,SAAS,CAAC;QACjC,IAAI,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,OAAO,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;QACnE,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,4BAA4B,kBAAkB,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACzG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;gBAC3B,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAkE,CAAC;YAC9F,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,UAAU,CAAC;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YAChF,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAC/B,OAAO,QAAQ,IAAI,SAAS,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YAC3B,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,OAAO,iBAAiB;IAC5B,KAAK,CAAC,KAAK,CAAC,IAA6B;QACvC,OAAO,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,IAA6B;QACjD,MAAM,OAAO,GAAG,IAAI,CAAC,OAA6B,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAA8B,CAAC;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,KAA2B,CAAC;QAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,MAA4B,CAAC;QAEjD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,OAAO;YAClB,CAAC,CAAC,2CAA2C,OAAO,OAAO;YAC3D,CAAC,CAAC,WAAW,MAAM,OAAO,CAAC;QAC7B,MAAM,OAAO,GAA2B,OAAO;YAC7C,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE;YACvE,CAAC,CAAC;gBACE,aAAa,EAAE,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;gBAClF,MAAM,EAAE,kBAAkB;aAC3B,CAAC;QAEN,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,OAAO;YACvB,CAAC,CAAC,CAAC,QAAQ,IAAI,2CAA2C,OAAO,EAAE,CAAC;YACpE,CAAC,CAAC,WAAW,MAAM,EAAE,CAAC;QACxB,MAAM,WAAW,GAAG,0BAA0B,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAE9D,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,MAA0B,CAAC;QAC/B,IAAI,QAA4B,CAAC;QAEjC,OAAO,KAAK,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;YACrE,MAAM,GAAG,GACP,GAAG,IAAI,uBAAuB,QAAQ,sBAAsB;gBAC5D,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YAC1C,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG;oBAAE,MAAM,IAAI,gBAAgB,CAAC,YAAY,CAAC,CAAC;gBACjE,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CACb,oGAAoG;wBAClG,iGAAiG,CACpG,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,KAAK,CACb,0BAA0B,GAAG,CAAC,MAAM,MAAM,OAAO,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,sCAAsC,EAAE,CACzH,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAgF,CAAC;YAC/G,QAAQ,GAAG,QAAQ,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,GAAG,SAAS,OAAO,CAAC;YAEhE,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;gBACtC,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK;oBAAE,MAAM;gBACjC,OAAO,IAAI,CAAC,CAAC;gBACb,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;gBACjD,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;gBACpD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC;gBACvC,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,QAAQ,GAAG,KAAK,EAAE,CAAC;gBACzE,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChD,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;gBAC5D,KAAK,CAAC,IAAI,CAAC;oBACT,UAAU,EAAE,OAAO;oBACnB,QAAQ,EAAE,YAAY;oBACtB,QAAQ,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;oBAC7D,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC/C,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC9B,CAAC,CAAC;YACL,CAAC;YAED,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAC3C,IAAI,CAAC,MAAM;gBAAE,MAAM;QACrB,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC;IAC7F,CAAC;CACF"}
|
package/dist/fetchers/git.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ConnectorFetcher, ConnectorFetcherOptions, FetcherItem } from '../types/fetcher.js';
|
|
1
|
+
import type { ConnectorFetcher, ConnectorFetcherOptions, FetcherItem, FetchResult } from '../types/fetcher.js';
|
|
2
2
|
import { type GitCommit } from './util/git.js';
|
|
3
3
|
export type { GitCommit } from './util/git.js';
|
|
4
4
|
/**
|
|
@@ -19,5 +19,6 @@ export declare class GitFetcher implements ConnectorFetcher {
|
|
|
19
19
|
private readonly source;
|
|
20
20
|
constructor(source: GitCommitSource);
|
|
21
21
|
fetch(opts: ConnectorFetcherOptions): Promise<FetcherItem[]>;
|
|
22
|
+
fetchWithReport(opts: ConnectorFetcherOptions): Promise<FetchResult>;
|
|
22
23
|
}
|
|
23
24
|
//# sourceMappingURL=git.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"git.d.ts","sourceRoot":"","sources":["../../src/fetchers/git.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"git.d.ts","sourceRoot":"","sources":["../../src/fetchers/git.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAC/G,OAAO,EAAsC,KAAK,SAAS,EAAE,MAAM,eAAe,CAAC;AAGnF,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAE/C;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,gBAAgB,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAChE,YAAY,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC;CACpD;AAED;;;GAGG;AACH,qBAAa,UAAW,YAAW,gBAAgB;IACrC,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,eAAe;IAE9C,KAAK,CAAC,IAAI,EAAE,uBAAuB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAI5D,eAAe,CAAC,IAAI,EAAE,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;CAkB3E"}
|
package/dist/fetchers/git.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { buildCommitUrl, formatCommitAsText } from './util/git.js';
|
|
2
|
+
import { toIsoOrUndefined } from './util/time.js';
|
|
2
3
|
/**
|
|
3
4
|
* Read-only local-git fetcher: maps decision-relevant commits to FetcherItems.
|
|
4
5
|
* Author = the commit author ("who to talk to"). `token` is unused (local).
|
|
@@ -9,19 +10,25 @@ export class GitFetcher {
|
|
|
9
10
|
this.source = source;
|
|
10
11
|
}
|
|
11
12
|
async fetch(opts) {
|
|
13
|
+
return (await this.fetchWithReport(opts)).items;
|
|
14
|
+
}
|
|
15
|
+
async fetchWithReport(opts) {
|
|
12
16
|
const limit = opts.limit ?? 100;
|
|
13
17
|
const commits = await this.source.getCommitHistory({ limit });
|
|
14
18
|
const remoteUrl = await this.source.getRemoteUrl();
|
|
15
|
-
|
|
19
|
+
const items = commits.map((c) => {
|
|
16
20
|
const url = buildCommitUrl(remoteUrl, c.sha);
|
|
21
|
+
const createdAt = toIsoOrUndefined(c.date);
|
|
17
22
|
return {
|
|
18
23
|
source_url: url,
|
|
19
24
|
platform: 'git',
|
|
20
25
|
raw_text: formatCommitAsText(c, url),
|
|
21
26
|
title: c.subject,
|
|
27
|
+
...(createdAt ? { created_at: createdAt } : {}),
|
|
22
28
|
...(c.author ? { author: { name: c.author } } : {}),
|
|
23
29
|
};
|
|
24
30
|
});
|
|
31
|
+
return { items, report: { platform: 'git', scanned: commits.length, requested: limit, skips: [] } };
|
|
25
32
|
}
|
|
26
33
|
}
|
|
27
34
|
//# sourceMappingURL=git.js.map
|
package/dist/fetchers/git.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"git.js","sourceRoot":"","sources":["../../src/fetchers/git.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAkB,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"git.js","sourceRoot":"","sources":["../../src/fetchers/git.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAkB,MAAM,eAAe,CAAC;AACnF,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAalD;;;GAGG;AACH,MAAM,OAAO,UAAU;IACQ;IAA7B,YAA6B,MAAuB;QAAvB,WAAM,GAAN,MAAM,CAAiB;IAAG,CAAC;IAExD,KAAK,CAAC,KAAK,CAAC,IAA6B;QACvC,OAAO,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,IAA6B;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC;QAChC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9D,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC9B,MAAM,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;YAC7C,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC3C,OAAO;gBACL,UAAU,EAAE,GAAG;gBACf,QAAQ,EAAE,KAAK;gBACf,QAAQ,EAAE,kBAAkB,CAAC,CAAC,EAAE,GAAG,CAAC;gBACpC,KAAK,EAAE,CAAC,CAAC,OAAO;gBAChB,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/C,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9B,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC;IACtG,CAAC;CACF"}
|
|
@@ -1,10 +1,23 @@
|
|
|
1
|
-
import type { ConnectorFetcher, ConnectorFetcherOptions, FetcherItem } from '../types/fetcher.js';
|
|
1
|
+
import type { ConnectorFetcher, ConnectorFetcherOptions, FetcherItem, FetchResult } from '../types/fetcher.js';
|
|
2
2
|
/**
|
|
3
|
-
* Read-only personal GitHub fetcher
|
|
4
|
-
*
|
|
5
|
-
* `
|
|
3
|
+
* Read-only personal GitHub fetcher.
|
|
4
|
+
*
|
|
5
|
+
* ALI-805: the old two queries (`author:+is:merged`, `commenter:`) missed
|
|
6
|
+
* self-filed issues (`commenter:` never matches the original post), every
|
|
7
|
+
* open or reverted PR (`is:merged` drops both - and a reversal is one of the
|
|
8
|
+
* most decision-dense things in a repo), and anything only assigned or
|
|
9
|
+
* mentioned. `involves:` is GitHub's own union of author/assignee/mentions/
|
|
10
|
+
* commenter, so it replaces both narrow queries - but it does NOT cover code
|
|
11
|
+
* review (confirmed against GitHub's search-qualifiers docs), so a PR the
|
|
12
|
+
* user only reviewed needs its own `reviewed-by:` query.
|
|
13
|
+
*
|
|
14
|
+
* And per item, this now fetches the discussion - issue/PR comments, PR
|
|
15
|
+
* review bodies, and inline review comments - not just the title and body.
|
|
16
|
+
* That is where the "why" usually lives: the body says what changed, the
|
|
17
|
+
* thread is where someone objects and the author agrees or pushes back.
|
|
6
18
|
*/
|
|
7
19
|
export declare class GitHubFetcher implements ConnectorFetcher {
|
|
8
20
|
fetch(opts: ConnectorFetcherOptions): Promise<FetcherItem[]>;
|
|
21
|
+
fetchWithReport(opts: ConnectorFetcherOptions): Promise<FetchResult>;
|
|
9
22
|
}
|
|
10
23
|
//# sourceMappingURL=github.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../src/fetchers/github.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../src/fetchers/github.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AA0L/G;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,aAAc,YAAW,gBAAgB;IAC9C,KAAK,CAAC,IAAI,EAAE,uBAAuB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAI5D,eAAe,CAAC,IAAI,EAAE,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;CAkC3E"}
|
package/dist/fetchers/github.js
CHANGED
|
@@ -1,7 +1,18 @@
|
|
|
1
1
|
import { fetch } from 'undici';
|
|
2
|
+
import { toIsoOrUndefined } from './util/time.js';
|
|
2
3
|
// GitHub's Search API returns at most 100 results/page and 1000 total (10 pages).
|
|
3
4
|
const GH_PER_PAGE_MAX = 100;
|
|
4
5
|
const GH_SEARCH_MAX_PAGES = 10;
|
|
6
|
+
// How many items may have their discussion (comments/reviews) in flight at
|
|
7
|
+
// once. Each item costs up to 3 extra requests (issue comments, PR reviews,
|
|
8
|
+
// PR review comments) - bounding this keeps a large personal history from
|
|
9
|
+
// blasting past GitHub's rate limit. Lower than the gateway's hosted scan
|
|
10
|
+
// (20, services/gateway/src/discover/githubHistorical.ts) because this runs
|
|
11
|
+
// unattended on a personal token with no retry/backoff around it.
|
|
12
|
+
const PARALLEL_DISCUSSION_FETCHES = 5;
|
|
13
|
+
// A single comment/review section is capped so one pathological thread (a
|
|
14
|
+
// bot dump, a copy-pasted log) can't blow the whole item's extraction budget.
|
|
15
|
+
const MAX_SECTION_CHARS = 4000;
|
|
5
16
|
/** Page through a GitHub search query until `target` items are collected (or the
|
|
6
17
|
* results run out / the 1000-result ceiling is hit). */
|
|
7
18
|
async function searchAll(query, headers, target) {
|
|
@@ -19,13 +30,137 @@ async function searchAll(query, headers, target) {
|
|
|
19
30
|
}
|
|
20
31
|
return out.slice(0, target);
|
|
21
32
|
}
|
|
33
|
+
/** Alternate between two ordered lists (PRs, issues) so neither can crowd the
|
|
34
|
+
* other out once the combined total is trimmed to `limit`. A plain
|
|
35
|
+
* concat-then-slice would let a prolific PR history exhaust the limit before
|
|
36
|
+
* a single issue is considered. */
|
|
37
|
+
function interleave(a, b, limit) {
|
|
38
|
+
const out = [];
|
|
39
|
+
let i = 0;
|
|
40
|
+
let j = 0;
|
|
41
|
+
while (out.length < limit && (i < a.length || j < b.length)) {
|
|
42
|
+
if (i < a.length)
|
|
43
|
+
out.push({ kind: 'pr', row: a[i++] });
|
|
44
|
+
if (out.length < limit && j < b.length)
|
|
45
|
+
out.push({ kind: 'issue', row: b[j++] });
|
|
46
|
+
}
|
|
47
|
+
return out;
|
|
48
|
+
}
|
|
49
|
+
/** Bounded-concurrency map that preserves input order in the output, so a
|
|
50
|
+
* fast item never jumps ahead of a slow one in the returned list. */
|
|
51
|
+
async function mapWithConcurrency(items, concurrency, fn) {
|
|
52
|
+
const results = new Array(items.length);
|
|
53
|
+
let next = 0;
|
|
54
|
+
async function worker() {
|
|
55
|
+
for (;;) {
|
|
56
|
+
const i = next++;
|
|
57
|
+
if (i >= items.length)
|
|
58
|
+
return;
|
|
59
|
+
results[i] = await fn(items[i]);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
await Promise.all(Array.from({ length: Math.max(1, Math.min(concurrency, items.length)) }, () => worker()));
|
|
63
|
+
return results;
|
|
64
|
+
}
|
|
65
|
+
function capSection(heading, entries) {
|
|
66
|
+
if (entries.length === 0)
|
|
67
|
+
return '';
|
|
68
|
+
let body = entries.join('\n\n');
|
|
69
|
+
if (body.length > MAX_SECTION_CHARS) {
|
|
70
|
+
body = `${body.slice(0, MAX_SECTION_CHARS)}\n[discussion truncated]`;
|
|
71
|
+
}
|
|
72
|
+
return `\n\n## ${heading}\n${body}`;
|
|
73
|
+
}
|
|
74
|
+
function formatEntries(entries, withState) {
|
|
75
|
+
return entries
|
|
76
|
+
.filter((e) => e.body?.trim())
|
|
77
|
+
.map((e) => {
|
|
78
|
+
const who = e.user?.login ?? 'Unknown';
|
|
79
|
+
const when = e.submitted_at ?? e.created_at ?? '';
|
|
80
|
+
const state = withState && e.state ? ` [${e.state}]` : '';
|
|
81
|
+
return `[${who}] (${when})${state}:\n${e.body}`;
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
/** Fetch one discussion endpoint, tolerating a failed or throwing request so
|
|
85
|
+
* one bad section never drops an otherwise-good item (ALI-805: the argument
|
|
86
|
+
* is a bonus on top of the announcement, not a requirement for it). */
|
|
87
|
+
async function fetchSection(url, headers, heading, withState) {
|
|
88
|
+
try {
|
|
89
|
+
const res = await fetch(url, { headers });
|
|
90
|
+
if (!res.ok)
|
|
91
|
+
return '';
|
|
92
|
+
const entries = (await res.json());
|
|
93
|
+
return capSection(heading, formatEntries(entries, withState));
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
return '';
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
function repoOf(item) {
|
|
100
|
+
return (item.repository_url ?? '').replace('https://api.github.com/repos/', '');
|
|
101
|
+
}
|
|
102
|
+
async function buildPrItem(pr, headers) {
|
|
103
|
+
const repo = repoOf(pr);
|
|
104
|
+
const status = pr.pull_request?.merged_at ? 'merged' : pr.state;
|
|
105
|
+
let rawText = `${pr.title}\n\n${pr.body ?? ''}\n\nStatus: ${status}\nRepo: ${repo}`.trim();
|
|
106
|
+
if (repo && pr.number != null) {
|
|
107
|
+
// Sequential, not Promise.all: GitHub's own best-practices guidance is
|
|
108
|
+
// to make requests serially rather than concurrently to avoid secondary
|
|
109
|
+
// rate limiting. PARALLEL_DISCUSSION_FETCHES already bounds how many
|
|
110
|
+
// ITEMS run at once - firing 3 more requests concurrently per item would
|
|
111
|
+
// undo that bound (5 items x 3 requests = 15 requests in flight).
|
|
112
|
+
rawText += await fetchSection(`https://api.github.com/repos/${repo}/issues/${pr.number}/comments?per_page=20`, headers, 'Comments', false);
|
|
113
|
+
rawText += await fetchSection(`https://api.github.com/repos/${repo}/pulls/${pr.number}/reviews?per_page=20`, headers, 'Code Reviews', true);
|
|
114
|
+
rawText += await fetchSection(`https://api.github.com/repos/${repo}/pulls/${pr.number}/comments?per_page=20`, headers, 'Review Comments', false);
|
|
115
|
+
}
|
|
116
|
+
const createdAt = toIsoOrUndefined(pr.created_at);
|
|
117
|
+
return {
|
|
118
|
+
source_url: pr.html_url,
|
|
119
|
+
platform: 'github',
|
|
120
|
+
raw_text: rawText,
|
|
121
|
+
title: pr.title,
|
|
122
|
+
...(createdAt ? { created_at: createdAt } : {}),
|
|
123
|
+
...(pr.user ? { author: { name: pr.user.login, handle: pr.user.login, url: pr.user.html_url } } : {}),
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
async function buildIssueItem(issue, headers) {
|
|
127
|
+
const repo = repoOf(issue);
|
|
128
|
+
let rawText = `${issue.title}\n\n${issue.body ?? ''}\n\nStatus: ${issue.state}`.trim();
|
|
129
|
+
if (repo && issue.number != null) {
|
|
130
|
+
rawText += await fetchSection(`https://api.github.com/repos/${repo}/issues/${issue.number}/comments?per_page=20`, headers, 'Comments', false);
|
|
131
|
+
}
|
|
132
|
+
const createdAt = toIsoOrUndefined(issue.created_at);
|
|
133
|
+
return {
|
|
134
|
+
source_url: issue.html_url,
|
|
135
|
+
platform: 'github',
|
|
136
|
+
raw_text: rawText,
|
|
137
|
+
title: issue.title,
|
|
138
|
+
...(createdAt ? { created_at: createdAt } : {}),
|
|
139
|
+
...(issue.user ? { author: { name: issue.user.login, handle: issue.user.login, url: issue.user.html_url } } : {}),
|
|
140
|
+
};
|
|
141
|
+
}
|
|
22
142
|
/**
|
|
23
|
-
* Read-only personal GitHub fetcher
|
|
24
|
-
*
|
|
25
|
-
* `
|
|
143
|
+
* Read-only personal GitHub fetcher.
|
|
144
|
+
*
|
|
145
|
+
* ALI-805: the old two queries (`author:+is:merged`, `commenter:`) missed
|
|
146
|
+
* self-filed issues (`commenter:` never matches the original post), every
|
|
147
|
+
* open or reverted PR (`is:merged` drops both - and a reversal is one of the
|
|
148
|
+
* most decision-dense things in a repo), and anything only assigned or
|
|
149
|
+
* mentioned. `involves:` is GitHub's own union of author/assignee/mentions/
|
|
150
|
+
* commenter, so it replaces both narrow queries - but it does NOT cover code
|
|
151
|
+
* review (confirmed against GitHub's search-qualifiers docs), so a PR the
|
|
152
|
+
* user only reviewed needs its own `reviewed-by:` query.
|
|
153
|
+
*
|
|
154
|
+
* And per item, this now fetches the discussion - issue/PR comments, PR
|
|
155
|
+
* review bodies, and inline review comments - not just the title and body.
|
|
156
|
+
* That is where the "why" usually lives: the body says what changed, the
|
|
157
|
+
* thread is where someone objects and the author agrees or pushes back.
|
|
26
158
|
*/
|
|
27
159
|
export class GitHubFetcher {
|
|
28
160
|
async fetch(opts) {
|
|
161
|
+
return (await this.fetchWithReport(opts)).items;
|
|
162
|
+
}
|
|
163
|
+
async fetchWithReport(opts) {
|
|
29
164
|
const headers = {
|
|
30
165
|
Authorization: `Bearer ${opts.token}`,
|
|
31
166
|
Accept: 'application/vnd.github+json',
|
|
@@ -37,31 +172,20 @@ export class GitHubFetcher {
|
|
|
37
172
|
}
|
|
38
173
|
const user = (await userRes.json());
|
|
39
174
|
const limit = opts.limit ?? 100;
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
for (const issue of issues) {
|
|
55
|
-
items.push({
|
|
56
|
-
source_url: issue.html_url,
|
|
57
|
-
platform: 'github',
|
|
58
|
-
raw_text: `${issue.title}\n\n${issue.body ?? ''}\n\nStatus: ${issue.state}`.trim(),
|
|
59
|
-
title: issue.title,
|
|
60
|
-
...(issue.user ? { author: { name: issue.user.login, handle: issue.user.login, url: issue.user.html_url } } : {}),
|
|
61
|
-
});
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
return items.slice(0, limit);
|
|
175
|
+
const [involvesPrs, reviewedPrs, issues] = await Promise.all([
|
|
176
|
+
searchAll(`https://api.github.com/search/issues?q=involves:${user.login}+type:pr`, headers, limit),
|
|
177
|
+
searchAll(`https://api.github.com/search/issues?q=reviewed-by:${user.login}+type:pr`, headers, limit),
|
|
178
|
+
searchAll(`https://api.github.com/search/issues?q=involves:${user.login}+type:issue`, headers, limit),
|
|
179
|
+
]);
|
|
180
|
+
// involves: and reviewed-by: can both return the same PR (e.g. you
|
|
181
|
+
// authored it AND someone else reviewed you on it too) - dedupe before
|
|
182
|
+
// the item ever reaches the discussion-fetch stage.
|
|
183
|
+
const prByUrl = new Map();
|
|
184
|
+
for (const pr of [...involvesPrs, ...reviewedPrs])
|
|
185
|
+
prByUrl.set(pr.html_url, pr);
|
|
186
|
+
const rows = interleave([...prByUrl.values()], issues, limit);
|
|
187
|
+
const items = await mapWithConcurrency(rows, PARALLEL_DISCUSSION_FETCHES, (r) => r.kind === 'pr' ? buildPrItem(r.row, headers) : buildIssueItem(r.row, headers));
|
|
188
|
+
return { items, report: { platform: 'github', scanned: rows.length, requested: limit, skips: [] } };
|
|
65
189
|
}
|
|
66
190
|
}
|
|
67
191
|
//# sourceMappingURL=github.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/fetchers/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/fetchers/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAE/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAwBlD,kFAAkF;AAClF,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,2EAA2E;AAC3E,4EAA4E;AAC5E,0EAA0E;AAC1E,0EAA0E;AAC1E,4EAA4E;AAC5E,kEAAkE;AAClE,MAAM,2BAA2B,GAAG,CAAC,CAAC;AAEtC,0EAA0E;AAC1E,8EAA8E;AAC9E,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAE/B;yDACyD;AACzD,KAAK,UAAU,SAAS,CAAC,KAAa,EAAE,OAA+B,EAAE,MAAc;IACrF,MAAM,GAAG,GAAuB,EAAE,CAAC;IACnC,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,MAAM,IAAI,IAAI,IAAI,mBAAmB,EAAE,IAAI,EAAE,EAAE,CAAC;QAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QAC/D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,KAAK,0BAA0B,OAAO,SAAS,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QAC/F,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM;QACnB,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAmC,CAAC;QAClE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/B,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;QACnB,IAAI,KAAK,CAAC,MAAM,GAAG,OAAO;YAAE,MAAM,CAAC,YAAY;IACjD,CAAC;IACD,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;AAC9B,CAAC;AAED;;;oCAGoC;AACpC,SAAS,UAAU,CAAO,CAAM,EAAE,CAAM,EAAE,KAAa;IACrD,MAAM,GAAG,GAA8D,EAAE,CAAC;IAC1E,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,GAAG,CAAC,MAAM,GAAG,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5D,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAE,EAAE,CAAC,CAAC;QACzD,IAAI,GAAG,CAAC,MAAM,GAAG,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAE,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;sEACsE;AACtE,KAAK,UAAU,kBAAkB,CAC/B,KAAU,EACV,WAAmB,EACnB,EAA2B;IAE3B,MAAM,OAAO,GAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,UAAU,MAAM;QACnB,SAAS,CAAC;YACR,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACjB,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAC9B,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC5G,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,UAAU,CAAC,OAAe,EAAE,OAAiB;IACpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACpC,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,IAAI,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACpC,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,0BAA0B,CAAC;IACvE,CAAC;IACD,OAAO,UAAU,OAAO,KAAK,IAAI,EAAE,CAAC;AACtC,CAAC;AAED,SAAS,aAAa,CAAC,OAA0B,EAAE,SAAkB;IACnE,OAAO,OAAO;SACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;SAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,IAAI,SAAS,CAAC;QACvC,MAAM,IAAI,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,OAAO,IAAI,GAAG,MAAM,IAAI,IAAI,KAAK,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAClD,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;wEAEwE;AACxE,KAAK,UAAU,YAAY,CACzB,GAAW,EACX,OAA+B,EAC/B,OAAe,EACf,SAAkB;IAElB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAsB,CAAC;QACxD,OAAO,UAAU,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;IAChE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,MAAM,CAAC,IAAsB;IACpC,OAAO,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,+BAA+B,EAAE,EAAE,CAAC,CAAC;AAClF,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,EAAoB,EAAE,OAA+B;IAC9E,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;IACxB,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC;IAChE,IAAI,OAAO,GAAG,GAAG,EAAE,CAAC,KAAK,OAAO,EAAE,CAAC,IAAI,IAAI,EAAE,eAAe,MAAM,WAAW,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAE3F,IAAI,IAAI,IAAI,EAAE,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;QAC9B,uEAAuE;QACvE,wEAAwE;QACxE,qEAAqE;QACrE,yEAAyE;QACzE,kEAAkE;QAClE,OAAO,IAAI,MAAM,YAAY,CAAC,gCAAgC,IAAI,WAAW,EAAE,CAAC,MAAM,uBAAuB,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QAC3I,OAAO,IAAI,MAAM,YAAY,CAAC,gCAAgC,IAAI,UAAU,EAAE,CAAC,MAAM,sBAAsB,EAAE,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;QAC5I,OAAO,IAAI,MAAM,YAAY,CAAC,gCAAgC,IAAI,UAAU,EAAE,CAAC,MAAM,uBAAuB,EAAE,OAAO,EAAE,iBAAiB,EAAE,KAAK,CAAC,CAAC;IACnJ,CAAC;IAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;IAClD,OAAO;QACL,UAAU,EAAE,EAAE,CAAC,QAAQ;QACvB,QAAQ,EAAE,QAAQ;QAClB,QAAQ,EAAE,OAAO;QACjB,KAAK,EAAE,EAAE,CAAC,KAAK;QACf,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtG,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,KAAuB,EAAE,OAA+B;IACpF,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3B,IAAI,OAAO,GAAG,GAAG,KAAK,CAAC,KAAK,OAAO,KAAK,CAAC,IAAI,IAAI,EAAE,eAAe,KAAK,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;IAEvF,IAAI,IAAI,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;QACjC,OAAO,IAAI,MAAM,YAAY,CAC3B,gCAAgC,IAAI,WAAW,KAAK,CAAC,MAAM,uBAAuB,EAClF,OAAO,EACP,UAAU,EACV,KAAK,CACN,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACrD,OAAO;QACL,UAAU,EAAE,KAAK,CAAC,QAAQ;QAC1B,QAAQ,EAAE,QAAQ;QAClB,QAAQ,EAAE,OAAO;QACjB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClH,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,aAAa;IACxB,KAAK,CAAC,KAAK,CAAC,IAA6B;QACvC,OAAO,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,IAA6B;QACjD,MAAM,OAAO,GAAG;YACd,aAAa,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE;YACrC,MAAM,EAAE,6BAA6B;YACrC,sBAAsB,EAAE,YAAY;SACrC,CAAC;QAEF,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,6BAA6B,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,uBAAuB,OAAO,CAAC,MAAM,uCAAuC,CAAC,CAAC;QAChG,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,CAAsB,CAAC;QAEzD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC;QAEhC,MAAM,CAAC,WAAW,EAAE,WAAW,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,SAAS,CAAC,mDAAmD,IAAI,CAAC,KAAK,UAAU,EAAE,OAAO,EAAE,KAAK,CAAC;YAClG,SAAS,CAAC,sDAAsD,IAAI,CAAC,KAAK,UAAU,EAAE,OAAO,EAAE,KAAK,CAAC;YACrG,SAAS,CAAC,mDAAmD,IAAI,CAAC,KAAK,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC;SACtG,CAAC,CAAC;QAEH,mEAAmE;QACnE,uEAAuE;QACvE,oDAAoD;QACpD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;QACpD,KAAK,MAAM,EAAE,IAAI,CAAC,GAAG,WAAW,EAAE,GAAG,WAAW,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAEhF,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAE9D,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,2BAA2B,EAAE,CAAC,CAAC,EAAE,EAAE,CAC9E,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,CAC/E,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC;IACtG,CAAC;CACF"}
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import type { ConnectorFetcher, ConnectorFetcherOptions, FetcherItem } from '../types/fetcher.js';
|
|
1
|
+
import type { ConnectorFetcher, ConnectorFetcherOptions, FetcherItem, FetchResult } from '../types/fetcher.js';
|
|
2
2
|
/**
|
|
3
|
-
* Read-only personal GitLab fetcher: the caller's merged merge requests
|
|
3
|
+
* Read-only personal GitLab fetcher: the caller's merged merge requests, paged
|
|
4
|
+
* by page number up to `limit` (a page shorter than requested is the last).
|
|
4
5
|
* `domain` (default gitlab.com) rides on the options index signature.
|
|
5
6
|
*/
|
|
6
7
|
export declare class GitLabFetcher implements ConnectorFetcher {
|
|
7
8
|
fetch(opts: ConnectorFetcherOptions): Promise<FetcherItem[]>;
|
|
9
|
+
fetchWithReport(opts: ConnectorFetcherOptions): Promise<FetchResult>;
|
|
8
10
|
}
|
|
9
11
|
//# sourceMappingURL=gitlab.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gitlab.d.ts","sourceRoot":"","sources":["../../src/fetchers/gitlab.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"gitlab.d.ts","sourceRoot":"","sources":["../../src/fetchers/gitlab.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,WAAW,EAAE,WAAW,EAAa,MAAM,qBAAqB,CAAC;AAe1H;;;;GAIG;AACH,qBAAa,aAAc,YAAW,gBAAgB;IAC9C,KAAK,CAAC,IAAI,EAAE,uBAAuB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAI5D,eAAe,CAAC,IAAI,EAAE,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;CAkD3E"}
|
package/dist/fetchers/gitlab.js
CHANGED
|
@@ -1,10 +1,18 @@
|
|
|
1
1
|
import { fetch } from 'undici';
|
|
2
|
+
import { toIsoOrUndefined } from './util/time.js';
|
|
3
|
+
// GitLab's own per-page maximum. Before ALI-828 the read was one page of at
|
|
4
|
+
// most 50, a cap nobody had chosen, whatever the caller asked for.
|
|
5
|
+
const GITLAB_PAGE_MAX = 100;
|
|
2
6
|
/**
|
|
3
|
-
* Read-only personal GitLab fetcher: the caller's merged merge requests
|
|
7
|
+
* Read-only personal GitLab fetcher: the caller's merged merge requests, paged
|
|
8
|
+
* by page number up to `limit` (a page shorter than requested is the last).
|
|
4
9
|
* `domain` (default gitlab.com) rides on the options index signature.
|
|
5
10
|
*/
|
|
6
11
|
export class GitLabFetcher {
|
|
7
12
|
async fetch(opts) {
|
|
13
|
+
return (await this.fetchWithReport(opts)).items;
|
|
14
|
+
}
|
|
15
|
+
async fetchWithReport(opts) {
|
|
8
16
|
const domain = opts.domain ?? 'gitlab.com';
|
|
9
17
|
const base = `https://${domain}/api/v4`;
|
|
10
18
|
const headers = { Authorization: `Bearer ${opts.token}` };
|
|
@@ -15,19 +23,40 @@ export class GitLabFetcher {
|
|
|
15
23
|
const user = (await userRes.json());
|
|
16
24
|
const limit = opts.limit ?? 100;
|
|
17
25
|
const items = [];
|
|
18
|
-
|
|
19
|
-
|
|
26
|
+
let scanned = 0;
|
|
27
|
+
let pagesUnreadable = 0;
|
|
28
|
+
// Constant for the whole run: `page` is an offset in units of per_page, so
|
|
29
|
+
// shrinking per_page on a later page moves the window backwards and re-reads
|
|
30
|
+
// rows already returned. The limit is enforced by stopping, not by the page.
|
|
31
|
+
const perPage = Math.min(limit, GITLAB_PAGE_MAX);
|
|
32
|
+
for (let page = 1; items.length < limit; page++) {
|
|
33
|
+
const mrRes = await fetch(`${base}/merge_requests?author_id=${user.id}&state=merged&per_page=${perPage}&page=${page}&order_by=updated_at`, { headers });
|
|
34
|
+
if (!mrRes.ok) {
|
|
35
|
+
// Before ALI-828 this returned nothing and said nothing.
|
|
36
|
+
pagesUnreadable += 1;
|
|
37
|
+
break;
|
|
38
|
+
}
|
|
20
39
|
const mrs = (await mrRes.json());
|
|
40
|
+
scanned += mrs.length;
|
|
21
41
|
for (const mr of mrs) {
|
|
42
|
+
if (items.length >= limit)
|
|
43
|
+
break;
|
|
44
|
+
const createdAt = toIsoOrUndefined(mr.created_at);
|
|
22
45
|
items.push({
|
|
23
46
|
source_url: mr.web_url,
|
|
24
47
|
platform: 'gitlab',
|
|
25
48
|
raw_text: `${mr.title}\n\n${mr.description ?? ''}\n\nStatus: ${mr.state}`.trim(),
|
|
26
49
|
title: mr.title,
|
|
50
|
+
...(createdAt ? { created_at: createdAt } : {}),
|
|
27
51
|
});
|
|
28
52
|
}
|
|
53
|
+
if (mrs.length < perPage)
|
|
54
|
+
break; // last page
|
|
29
55
|
}
|
|
30
|
-
|
|
56
|
+
const skips = [];
|
|
57
|
+
if (pagesUnreadable > 0)
|
|
58
|
+
skips.push({ kind: 'error', count: pagesUnreadable, detail: 'merge request pages the token could not read' });
|
|
59
|
+
return { items, report: { platform: 'gitlab', scanned, requested: limit, skips } };
|
|
31
60
|
}
|
|
32
61
|
}
|
|
33
62
|
//# sourceMappingURL=gitlab.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gitlab.js","sourceRoot":"","sources":["../../src/fetchers/gitlab.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"gitlab.js","sourceRoot":"","sources":["../../src/fetchers/gitlab.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAE/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAUlD,4EAA4E;AAC5E,mEAAmE;AACnE,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B;;;;GAIG;AACH,MAAM,OAAO,aAAa;IACxB,KAAK,CAAC,KAAK,CAAC,IAA6B;QACvC,OAAO,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,IAA6B;QACjD,MAAM,MAAM,GAAI,IAAI,CAAC,MAA6B,IAAI,YAAY,CAAC;QACnE,MAAM,IAAI,GAAG,WAAW,MAAM,SAAS,CAAC;QACxC,MAAM,OAAO,GAAG,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;QAE1D,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,uBAAuB,OAAO,CAAC,MAAM,2CAA2C,CAAC,CAAC;QACpG,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,CAAmB,CAAC;QAEtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC;QAChC,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,eAAe,GAAG,CAAC,CAAC;QAExB,2EAA2E;QAC3E,6EAA6E;QAC7E,6EAA6E;QAC7E,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC;QACjD,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,MAAM,KAAK,CACvB,GAAG,IAAI,6BAA6B,IAAI,CAAC,EAAE,0BAA0B,OAAO,SAAS,IAAI,sBAAsB,EAC/G,EAAE,OAAO,EAAE,CACZ,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;gBACd,yDAAyD;gBACzD,eAAe,IAAI,CAAC,CAAC;gBACrB,MAAM;YACR,CAAC;YACD,MAAM,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAyB,CAAC;YACzD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;YACtB,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;gBACrB,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK;oBAAE,MAAM;gBACjC,MAAM,SAAS,GAAG,gBAAgB,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;gBAClD,KAAK,CAAC,IAAI,CAAC;oBACT,UAAU,EAAE,EAAE,CAAC,OAAO;oBACtB,QAAQ,EAAE,QAAQ;oBAClB,QAAQ,EAAE,GAAG,EAAE,CAAC,KAAK,OAAO,EAAE,CAAC,WAAW,IAAI,EAAE,eAAe,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE;oBAChF,KAAK,EAAE,EAAE,CAAC,KAAK;oBACf,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAChD,CAAC,CAAC;YACL,CAAC;YACD,IAAI,GAAG,CAAC,MAAM,GAAG,OAAO;gBAAE,MAAM,CAAC,YAAY;QAC/C,CAAC;QAED,MAAM,KAAK,GAAgB,EAAE,CAAC;QAC9B,IAAI,eAAe,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,8CAA8C,EAAE,CAAC,CAAC;QACvI,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;IACrF,CAAC;CACF"}
|
package/dist/fetchers/jira.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ConnectorFetcher, ConnectorFetcherOptions, FetcherItem } from '../types/fetcher.js';
|
|
1
|
+
import type { ConnectorFetcher, ConnectorFetcherOptions, FetcherItem, FetchResult } from '../types/fetcher.js';
|
|
2
2
|
/**
|
|
3
3
|
* Read-only personal Jira fetcher: issues assigned to or reported by the caller.
|
|
4
4
|
* OAuth (cloudId) or basic-auth (domain+email). Author = the issue reporter.
|
|
@@ -6,5 +6,6 @@ import type { ConnectorFetcher, ConnectorFetcherOptions, FetcherItem } from '../
|
|
|
6
6
|
*/
|
|
7
7
|
export declare class JiraFetcher implements ConnectorFetcher {
|
|
8
8
|
fetch(opts: ConnectorFetcherOptions): Promise<FetcherItem[]>;
|
|
9
|
+
fetchWithReport(opts: ConnectorFetcherOptions): Promise<FetchResult>;
|
|
9
10
|
}
|
|
10
11
|
//# sourceMappingURL=jira.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jira.d.ts","sourceRoot":"","sources":["../../src/fetchers/jira.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"jira.d.ts","sourceRoot":"","sources":["../../src/fetchers/jira.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AA4B/G;;;;GAIG;AACH,qBAAa,WAAY,YAAW,gBAAgB;IAC5C,KAAK,CAAC,IAAI,EAAE,uBAAuB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAI5D,eAAe,CAAC,IAAI,EAAE,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;CA8E3E"}
|
package/dist/fetchers/jira.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { fetch } from 'undici';
|
|
2
2
|
import { FetcherAuthError } from './errors.js';
|
|
3
|
+
import { toIsoOrUndefined } from './util/time.js';
|
|
3
4
|
// Jira Cloud's /search/jql caps maxResults at 100 per request and paginates via nextPageToken.
|
|
4
5
|
const JIRA_PAGE_MAX = 100;
|
|
5
6
|
function extractAdfText(adf) {
|
|
@@ -17,6 +18,9 @@ function extractAdfText(adf) {
|
|
|
17
18
|
*/
|
|
18
19
|
export class JiraFetcher {
|
|
19
20
|
async fetch(opts) {
|
|
21
|
+
return (await this.fetchWithReport(opts)).items;
|
|
22
|
+
}
|
|
23
|
+
async fetchWithReport(opts) {
|
|
20
24
|
const cloudId = opts.cloudId;
|
|
21
25
|
const siteBase = opts.siteBase;
|
|
22
26
|
const email = opts.email;
|
|
@@ -37,7 +41,8 @@ export class JiraFetcher {
|
|
|
37
41
|
const body = {
|
|
38
42
|
jql,
|
|
39
43
|
maxResults: Math.min(limit - issues.length, JIRA_PAGE_MAX),
|
|
40
|
-
|
|
44
|
+
// `created` must be asked for by name or the search API leaves it out.
|
|
45
|
+
fields: ['summary', 'description', 'status', 'key', 'reporter', 'created'],
|
|
41
46
|
...(nextPageToken ? { nextPageToken } : {}),
|
|
42
47
|
};
|
|
43
48
|
const res = await fetch(`${base}/rest/api/3/search/jql`, {
|
|
@@ -60,8 +65,9 @@ export class JiraFetcher {
|
|
|
60
65
|
nextPageToken = data.isLast ? undefined : data.nextPageToken;
|
|
61
66
|
} while (nextPageToken && issues.length < limit);
|
|
62
67
|
const browseBase = isOAuth ? (siteBase ?? `https://api.atlassian.com/ex/jira/${cloudId}`) : base;
|
|
63
|
-
|
|
68
|
+
const items = issues.slice(0, limit).map((issue) => {
|
|
64
69
|
const desc = extractAdfText(issue.fields.description);
|
|
70
|
+
const createdAt = toIsoOrUndefined(issue.fields.created);
|
|
65
71
|
return {
|
|
66
72
|
source_url: `${browseBase}/browse/${issue.key}`,
|
|
67
73
|
platform: 'jira',
|
|
@@ -73,6 +79,7 @@ export class JiraFetcher {
|
|
|
73
79
|
.filter(Boolean)
|
|
74
80
|
.join('\n\n'),
|
|
75
81
|
title: `[${issue.key}] ${issue.fields.summary}`,
|
|
82
|
+
...(createdAt ? { created_at: createdAt } : {}),
|
|
76
83
|
...(issue.fields.reporter?.displayName
|
|
77
84
|
? {
|
|
78
85
|
author: {
|
|
@@ -83,6 +90,7 @@ export class JiraFetcher {
|
|
|
83
90
|
: {}),
|
|
84
91
|
};
|
|
85
92
|
});
|
|
93
|
+
return { items, report: { platform: 'jira', scanned: issues.length, requested: limit, skips: [] } };
|
|
86
94
|
}
|
|
87
95
|
}
|
|
88
96
|
//# sourceMappingURL=jira.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jira.js","sourceRoot":"","sources":["../../src/fetchers/jira.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAE/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"jira.js","sourceRoot":"","sources":["../../src/fetchers/jira.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAE/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAelD,+FAA+F;AAC/F,MAAM,aAAa,GAAG,GAAG,CAAC;AAE1B,SAAS,cAAc,CAAC,GAAmF;IACzG,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,CAAC;IACpB,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;SACvB,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;SAC5E,IAAI,CAAC,GAAG,CAAC;SACT,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;;GAIG;AACH,MAAM,OAAO,WAAW;IACtB,KAAK,CAAC,KAAK,CAAC,IAA6B;QACvC,OAAO,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,IAA6B;QACjD,MAAM,OAAO,GAAG,IAAI,CAAC,OAA6B,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAA8B,CAAC;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,KAA2B,CAAC;QAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,MAA4B,CAAC;QAEjD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,qCAAqC,OAAO,EAAE,CAAC,CAAC,CAAC,WAAW,MAAM,EAAE,CAAC;QAC5F,MAAM,OAAO,GAA2B,OAAO;YAC7C,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE;YACvE,CAAC,CAAC;gBACE,aAAa,EAAE,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;gBAClF,MAAM,EAAE,kBAAkB;aAC3B,CAAC;QAEN,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC;QAChC,MAAM,GAAG,GAAG,4EAA4E,CAAC;QAEzF,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,IAAI,aAAiC,CAAC;QACtC,GAAG,CAAC;YACF,MAAM,IAAI,GAA4B;gBACpC,GAAG;gBACH,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC;gBAC1D,uEAAuE;gBACvE,MAAM,EAAE,CAAC,SAAS,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,CAAC;gBAC1E,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5C,CAAC;YACF,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,wBAAwB,EAAE;gBACvD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC3D,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG;oBAAE,MAAM,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;gBAC3D,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CACb,wFAAwF;wBACtF,iDAAiD,CACpD,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,CAAC,MAAM,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;YACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAuE,CAAC;YACtG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;YACpC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC;QAC/D,CAAC,QAAQ,aAAa,IAAI,MAAM,CAAC,MAAM,GAAG,KAAK,EAAE;QAEjD,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,qCAAqC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEjG,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACjD,MAAM,IAAI,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACtD,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,GAAG,UAAU,WAAW,KAAK,CAAC,GAAG,EAAE;gBAC/C,QAAQ,EAAE,MAAM;gBAChB,QAAQ,EAAE;oBACR,IAAI,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE;oBACxC,IAAI;oBACJ,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,WAAW,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE;iBACvE;qBACE,MAAM,CAAC,OAAO,CAAC;qBACf,IAAI,CAAC,MAAM,CAAC;gBACf,KAAK,EAAE,IAAI,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE;gBAC/C,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/C,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,WAAW;oBACpC,CAAC,CAAC;wBACE,MAAM,EAAE;4BACN,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW;4BACvC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;yBAC7F;qBACF;oBACH,CAAC,CAAC,EAAE,CAAC;aACc,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC;IACtG,CAAC;CACF"}
|