@preventive/triage 1.0.0-alpha.50 → 1.0.0-alpha.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/common/managed/finding-detail.d.ts +2 -1
- package/common/managed/finding-detail.js +42 -11
- package/common/managed/finding-values.d.ts +2 -0
- package/common/managed/finding-values.js +44 -0
- package/common/managed/lite-name.d.ts +2 -0
- package/common/managed/lite-name.js +20 -0
- package/common/managed/npm-packages.d.ts +2 -0
- package/common/managed/npm-packages.js +18 -1
- package/common/managed/routes.js +19 -7
- package/common/managed/team-app.d.ts +1 -0
- package/common/managed/team-app.js +17 -1
- package/out/activity.js +4 -4
- package/out/{chunk-75YLHBCM.js → chunk-2K6UN3PC.js} +1 -1
- package/out/{chunk-HTBYHSPG.js → chunk-CHRBUGVN.js} +1 -1
- package/out/{chunk-4XMP7YVH.js → chunk-FMXWXDR4.js} +1 -1
- package/out/chunk-HT2SSA3V.js +1 -0
- package/out/{chunk-2WDZPXVL.js → chunk-IMM274IW.js} +1 -1
- package/out/{chunk-FKBXNUW4.js → chunk-KOGNQDSJ.js} +1 -1
- package/out/chunk-LJ7XRXFU.js +52 -0
- package/out/{chunk-PBZPYIG7.js → chunk-MYGJBYYJ.js} +1 -1
- package/out/chunk-NCN3M5EP.js +3 -0
- package/out/chunk-OGAYMEER.js +232 -0
- package/out/client-managed-import.js +1 -1
- package/out/client-managed.js +26 -26
- package/out/compare.js +14 -14
- package/out/dialogs.js +31 -31
- package/out/graph.js +1 -1
- package/out/index.html +7 -7
- package/out/scan.js +2 -2
- package/out/view.css +1 -1
- package/out/view.js +11 -11
- package/package.json +8 -1
- package/server-managed/agent-merges.ts +1 -1
- package/server-managed/app-agents.ts +4 -2
- package/server-managed/commit-lockfiles.ts +7 -4
- package/server-managed/comparison.ts +13 -2
- package/server-managed/db-neon.ts +13 -3
- package/server-managed/db.ts +2 -1
- package/server-managed/dependency-review-comments.ts +16 -18
- package/server-managed/dependency-reviews.ts +45 -27
- package/server-managed/github-app.ts +3 -42
- package/server-managed/github-requests.ts +127 -0
- package/server-managed/http.ts +433 -129
- package/server-managed/listed-activity.ts +39 -0
- package/server-managed/locked-packages.ts +28 -5
- package/server-managed/npm-kept.ts +18 -4
- package/server-managed/npm-patch.ts +325 -0
- package/server-managed/pull-dependencies.ts +9 -6
- package/server-managed/pull-dependency-reviews.ts +19 -13
- package/server-managed/repository-agents.ts +162 -69
- package/server-managed/repository-dependency-reviews.ts +6 -5
- package/server-managed/repository-pulls.ts +114 -58
- package/server-managed/repository-pulse.ts +5 -3
- package/server-managed/repository-stamp.ts +41 -0
- package/server-managed/reviewed-dependencies.ts +4 -5
- package/server-managed/team-app.ts +3 -1
- package/server-managed/team-reports.ts +29 -10
- package/out/chunk-6LNQTKAV.js +0 -52
- package/out/chunk-6S2U2UYG.js +0 -232
- package/out/chunk-FC65TPUB.js +0 -1
- package/out/chunk-UK7RGJT6.js +0 -3
package/server-managed/http.ts
CHANGED
|
@@ -101,12 +101,15 @@
|
|
|
101
101
|
// POST /api/admin/teams/{set,remove}-member → admin links/unlinks a user (+perms) | 401/403/404
|
|
102
102
|
// POST /api/admin/teams/set-dependency-review → admin toggles dependency review publication for a team | 400/401/403/404
|
|
103
103
|
// POST /api/admin/teams/set-npm-scopes → admin replaces a team's npm scopes | 400/401/403/404
|
|
104
|
-
// GET/POST /api/npm/reviews → read private/team reviews; save, publish, revoke or continue a set
|
|
104
|
+
// GET/POST /api/npm/reviews → read private/team reviews; save, publish, revoke or continue a set; a patched version's
|
|
105
|
+
// (patchHash) changed with where its patch is (patchRepoId, patchCommit, patchPath) | 400/401/403/404/409/422
|
|
105
106
|
// POST /api/npm/reviews/discard → author discards an unpublished review and its file rows | 400/401/403/409
|
|
106
107
|
// GET /api/npm/reviews/latest → latest own review for exact target bytes, across all bases | 400/401/403
|
|
107
108
|
// GET /api/npm/reviews/team?teamId=&beforeTime=&beforeId= → the reviews published to a team, the latest first, for one who
|
|
108
109
|
// reads them there | 400/401/404
|
|
109
110
|
// GET /api/npm/package?name=&version= → a published version's manifest and files | 400/401/404/413/429/502
|
|
111
|
+
// GET /api/npm/package?name=&version=&patchHash=&patchRepoId=&patchCommit=&patchPath= → its files as a pnpm patch of a
|
|
112
|
+
// repository the reader reads all of changes them | 400/401/403/404/409/413/422/429/502
|
|
110
113
|
// GET /api/npm/versions?name= → its dist-tags, versions, when each was published and its first was, and whether a team lists its scope | 400/401/404/429/502
|
|
111
114
|
// GET /api/npm/download?name=&version= → its tarball | 400/401/404/413/429/502
|
|
112
115
|
// GET /api/npm/stats?name= → its downloads over the last year and its GitHub repository's figures | 400/401/404/429/502
|
|
@@ -135,8 +138,9 @@ import { mergeLinkGroups } from './link-reports.ts'
|
|
|
135
138
|
import { backfillFindingIds, reportEntries } from '@preventive/report'
|
|
136
139
|
import type { IncomingMessage, ServerResponse } from 'node:http'
|
|
137
140
|
import { pipeline } from 'node:stream/promises'
|
|
138
|
-
import {
|
|
139
|
-
|
|
141
|
+
import { setTimeout as delay } from 'node:timers/promises'
|
|
142
|
+
import { type KeptStore, REPOSITORY_READ_KEPT_MS, authoredCommitStore, branchTipStore, codeHeadStore, codeTreeStore, commitPullStore, githubTagStore, historyChangeStore, historyPageStore, historySquashStore, historyTreeStore,
|
|
143
|
+
issueListStore, lockfileReadStore,
|
|
140
144
|
npmDeltaStore, npmPublishTimeStore, npmVersionStore, ownKeptPrunes, pullDependencyStore, pullListStore, repositoryReadStore, reviewBaselineStore, riskLineStore, scanReadStore } from './npm-kept.ts'
|
|
141
145
|
import { plainObject } from './npm-manifest.ts'
|
|
142
146
|
import { type NpmDelta, keptNpmDeltas, npmDelta } from './npm-delta.ts'
|
|
@@ -174,6 +178,7 @@ import { DEFAULT_MANAGED_SCAN_MODEL, MANAGED_SCAN_MODELS } from '../common/manag
|
|
|
174
178
|
import { CONFIG_PATH, type ManagedSessionInfo, type ServerInfo } from '../common/server-info.ts'
|
|
175
179
|
import { bundleCommitHash } from '../common/bundle-commit.js'
|
|
176
180
|
import { GithubApiError, collectRepos, fetchPublicRepository, installUrl, publicRepositoryName, repositoryInstallation } from './github-app.ts'
|
|
181
|
+
import { type TaggedRead, keepTaggedReadsIn } from './github-requests.ts'
|
|
177
182
|
import { upstreamGithub } from './github-errors.ts'
|
|
178
183
|
import { createClient } from '@preventive/upstream/github.js'
|
|
179
184
|
import type { ConnectedRepo } from './github-app.ts'
|
|
@@ -213,13 +218,14 @@ import { ManagedMutationError, reportReferenceSnapshot } from './management.ts'
|
|
|
213
218
|
import { BundleBuildError, buildRepositoryBundle, parseBundleBuild, withBundleBuildLease } from './bundle-build.ts'
|
|
214
219
|
import { NpmPackageError, type NpmReader, type NpmVersionDocument, type NpmVersionStore, VERSION_KEPT_MS, canReadPrivateNpm, npmFileRows, npmPublishTime, npmScopesCount, npmTarballFilename, readNpmVersion, readNpmVersions, readTogether } from './npm-packages.ts'
|
|
215
220
|
import { loadNpmFiles, loadNpmPackageBody, loadNpmTarball } from './npm-loads.ts'
|
|
221
|
+
import { type NpmPatch, type PatchedFile, patchedFiles, readPatch, requestedPatch } from './npm-patch.ts'
|
|
216
222
|
import { type RepositoryCodeKept, loadRepositoryCodeBody, repositoryCodeFits, repositoryCodeKept } from './repository-code.ts'
|
|
217
223
|
import { type ChangesStore, type CommitChanges, type CommitTree, TREES_READ, changesAsked, changesWithin, directoryFiles, repositoryChanges } from './repository-changes.ts'
|
|
218
224
|
import { COAUTHORS_READ, coauthorsOf } from './commit-coauthors.ts'
|
|
219
225
|
import { type AllHeads, type HistoryCommit, type HistoryPages, MAX_HISTORY_PAGE, historyFrom, historyStarts, openPullHeads,
|
|
220
226
|
repositoryAllHeads, repositoryAllHistory, repositoryHistory, repositoryRefTips } from './repository-history.ts'
|
|
221
227
|
import { type KnownMerges, type SquashedPull, type Squashes, squashedPulls } from './repository-squashes.ts'
|
|
222
|
-
import { AGENTS_READ_VERSION, COMMITS_CURSOR, type CommitPulls, repositoryAgents } from './repository-agents.ts'
|
|
228
|
+
import { AGENTS_READ_VERSION, type AuthoredCommit, type BranchTip, type BranchTips, COMMITS_CURSOR, type CommitPulls, repositoryAgents } from './repository-agents.ts'
|
|
223
229
|
import { ISSUES_CURSOR, ISSUES_READ_VERSION, repositoryIssues, repositoryItem } from './repository-issues.ts'
|
|
224
230
|
import { type GithubAsked, type ListedLockfile, type LockfileRead, type LockfileStore, MAX_LOCKED, PULL_DEPENDENCIES_READ, type PullCommits, type PullDependencies,
|
|
225
231
|
type PullDependencyStore, eachOf, lockfileFormat, pullDependenciesAt } from './pull-dependencies.ts'
|
|
@@ -230,10 +236,13 @@ import { type ReviewBaselineStore, baselineFor, heldDependencies, reviewedDepend
|
|
|
230
236
|
import { formattedPull, withFormatted } from './dependency-review-comments.ts'
|
|
231
237
|
import type { ReviewedDependencies } from './reviewed-dependencies.ts'
|
|
232
238
|
import { commentReviews } from './pull-comment-reviews.ts'
|
|
233
|
-
import { PULSE_WEEKS, PULSE_YEAR_VERSION, type PulseStats, StatsPending, commitStats, defaultHead, periodStats, pulseSince,
|
|
239
|
+
import { PULSE_WEEKS, PULSE_YEAR_VERSION, type PulseNarrowing, type PulseStats, StatsPending, commitStats, defaultHead, periodStats, pulseSince,
|
|
240
|
+
repositoryActivity, repositoryLines,
|
|
234
241
|
repositoryStats, withLines, yearCommits } from './repository-pulse.ts'
|
|
235
242
|
import { type AgentMerges, agentCommits, agentMerges, directCommits, sameMerges } from './agent-merges.ts'
|
|
236
|
-
import {
|
|
243
|
+
import { listedActivity } from './listed-activity.ts'
|
|
244
|
+
import { STAMP_READ, repositoryStamp } from './repository-stamp.ts'
|
|
245
|
+
import { ISSUES_LIST_READ_VERSION, type IssueList, type Listed, PULLS_READ_VERSION, type PullList, pullsReadTo, readIssues, readPulls } from './repository-pulls.ts'
|
|
237
246
|
import { AGENT_MARKS_READ } from './app-agents.ts'
|
|
238
247
|
import { releaseCommits } from './release-commits.ts'
|
|
239
248
|
import { computeBundleDiff } from '../common/bundle-compare-diff.js'
|
|
@@ -246,8 +255,8 @@ import { NpmRiskError } from '../risk/index.js'
|
|
|
246
255
|
import { type PrettyCache, PrettyError, type PrettyRequest, compared, prettyRequest, prettyText } from './pretty-print.ts'
|
|
247
256
|
import { renameDiffOf, renameDiffRequest, renamedPairs } from './rename-diff.ts'
|
|
248
257
|
import { typeOnlyFiles } from './type-only.ts'
|
|
249
|
-
import { comparisonBody, emptySide, npmComparedSide } from './comparison.ts'
|
|
250
|
-
import { isNpmPackageName, isNpmPackageSpec, npmPackageScope } from '../common/managed/npm-packages.js'
|
|
258
|
+
import { comparisonBody, emptySide, npmComparedSide, patchedVersionBody } from './comparison.ts'
|
|
259
|
+
import { isNpmPackageName, isNpmPackageSpec, isPatchHash, npmPackageScope } from '../common/managed/npm-packages.js'
|
|
251
260
|
|
|
252
261
|
const SESSION_PATH = '/api/auth/session'
|
|
253
262
|
const PROFILE_PATH = '/api/auth/profile'
|
|
@@ -1525,18 +1534,24 @@ type RepositoryReader = Awaited<ReturnType<ReturnType<typeof createRepositoryBro
|
|
|
1525
1534
|
type RepositoryContents = Awaited<ReturnType<typeof readRepositoryContents>>
|
|
1526
1535
|
type RepositoryRefs = Awaited<ReturnType<RepositoryReader['refs']>> & { defaultContents?: RepositoryContents | null }
|
|
1527
1536
|
|
|
1528
|
-
|
|
1537
|
+
// A directory's listing at a commit (`reader.directory`), which never changes,
|
|
1538
|
+
// as kept by them (repositoryTreesOf `listings`), else read.
|
|
1539
|
+
type DirectoryListing = Awaited<ReturnType<RepositoryReader['directory']>>
|
|
1540
|
+
type ListedAt = (commit: string, path: string, read: () => Promise<DirectoryListing>) => Promise<DirectoryListing>
|
|
1541
|
+
|
|
1542
|
+
async function readRepositoryContents(reader: RepositoryReader, ref: string, path: string, virtualEntries: RepositoryEntry[] | null, listed: ListedAt) {
|
|
1529
1543
|
const commit = await reader.commit(ref)
|
|
1530
|
-
const contents = virtualEntries ? { entries: virtualEntries, limited: false } : await reader.directory(path, commit)
|
|
1544
|
+
const contents = virtualEntries ? { entries: virtualEntries, limited: false } : await listed(commit, path, () => reader.directory(path, commit))
|
|
1531
1545
|
return { ...contents, path, commit }
|
|
1532
1546
|
}
|
|
1533
1547
|
|
|
1534
|
-
async function readRepositoryRefs(db: ManagedDb, reader: RepositoryReader, repo: SelectedRepo, withDefault: boolean, virtualEntries: RepositoryEntry[] | null
|
|
1548
|
+
async function readRepositoryRefs(db: ManagedDb, reader: RepositoryReader, repo: SelectedRepo, withDefault: boolean, virtualEntries: RepositoryEntry[] | null,
|
|
1549
|
+
listed: ListedAt): Promise<RepositoryRefs> {
|
|
1535
1550
|
// Wait for both, including failures: a deleted cached branch must not prevent
|
|
1536
1551
|
// a successful live default lookup from recovering the initial directory.
|
|
1537
1552
|
const [refs, contents] = await Promise.allSettled([
|
|
1538
1553
|
reader.refs(),
|
|
1539
|
-
withDefault && repo.cachedDefaultBranch ? readRepositoryContents(reader, `heads/${repo.cachedDefaultBranch}`, '', virtualEntries) : Promise.resolve(null),
|
|
1554
|
+
withDefault && repo.cachedDefaultBranch ? readRepositoryContents(reader, `heads/${repo.cachedDefaultBranch}`, '', virtualEntries, listed) : Promise.resolve(null),
|
|
1540
1555
|
])
|
|
1541
1556
|
if (refs.status === 'rejected') throw refs.reason
|
|
1542
1557
|
const defaultBranch = refs.value.defaultBranch || null
|
|
@@ -1544,7 +1559,7 @@ async function readRepositoryRefs(db: ManagedDb, reader: RepositoryReader, repo:
|
|
|
1544
1559
|
if (changed) await db.cacheRepoDefaultBranch(repo, defaultBranch)
|
|
1545
1560
|
if (!withDefault) return refs.value
|
|
1546
1561
|
if (!defaultBranch) return { ...refs.value, defaultContents: null }
|
|
1547
|
-
if (changed) return { ...refs.value, defaultContents: await readRepositoryContents(reader, `heads/${defaultBranch}`, '', virtualEntries) }
|
|
1562
|
+
if (changed) return { ...refs.value, defaultContents: await readRepositoryContents(reader, `heads/${defaultBranch}`, '', virtualEntries, listed) }
|
|
1548
1563
|
if (contents.status === 'rejected') throw contents.reason
|
|
1549
1564
|
return { ...refs.value, defaultContents: contents.value }
|
|
1550
1565
|
}
|
|
@@ -1613,6 +1628,9 @@ async function handleRepositoryBrowser(res: ServerResponse, deps: ManagedHttpDep
|
|
|
1613
1628
|
if (!asked) return
|
|
1614
1629
|
const { repoId, ref } = asked
|
|
1615
1630
|
const withDefault = refs && query.get('withDefault') === 'true'
|
|
1631
|
+
// The commit alone (`listing=none`), as a view reading none of its entries
|
|
1632
|
+
// opens at.
|
|
1633
|
+
const commitOnly = !refs && query.get('listing') === 'none'
|
|
1616
1634
|
const path = refs ? '' : asked.path
|
|
1617
1635
|
const authorize = async (handed = false) => {
|
|
1618
1636
|
const access = await authorizeRepository(res, deps, cookie, repoId, path, { handed })
|
|
@@ -1626,8 +1644,10 @@ async function handleRepositoryBrowser(res: ServerResponse, deps: ManagedHttpDep
|
|
|
1626
1644
|
const reader = await browser.reader(access.repo)
|
|
1627
1645
|
// Microseconds, so overlapping browses practically never share a listing time.
|
|
1628
1646
|
const observedUs = Math.round((performance.timeOrigin + performance.now()) * 1000)
|
|
1629
|
-
const
|
|
1630
|
-
const
|
|
1647
|
+
const trees = repositoryTreesOf(deps)
|
|
1648
|
+
const listedAt: ListedAt = (commit, at, read) => trees.listing(`${repoId}\0${LISTING_READ}\0${commit}\0${at}`, read)
|
|
1649
|
+
const revisions = refs ? await readRepositoryRefs(deps.db, reader, access.repo, withDefault, access.virtualEntries, listedAt) : null
|
|
1650
|
+
const contents = refs ? null : commitOnly ? { path, commit: await reader.commit(ref) } : await readRepositoryContents(reader, ref, path, access.virtualEntries, listedAt)
|
|
1631
1651
|
const { tagCommits, tagsComplete, ...listed } = revisions ?? { tagCommits: [], tagsComplete: false }
|
|
1632
1652
|
// Every refs read (Create a bundle's, and the code viewer's) refreshes the
|
|
1633
1653
|
// tags bundle catalogs show, and a tag read keeps the one its revision
|
|
@@ -1644,7 +1664,7 @@ async function handleRepositoryBrowser(res: ServerResponse, deps: ManagedHttpDep
|
|
|
1644
1664
|
if (!sameRepository(current.repo, access.repo)) { sendJson(res, 409, { error: 'repository-changed' }); return }
|
|
1645
1665
|
sendJson(res, 200, refs ? { ...listed, tagCommits, ...(withDefault ? {
|
|
1646
1666
|
defaultContents: scopeRepositoryContents(revisions?.defaultContents ?? null, current.virtualEntries),
|
|
1647
|
-
} : {}) } : scopeRepositoryContents(contents, current.virtualEntries))
|
|
1667
|
+
} : {}) } : commitOnly ? contents : scopeRepositoryContents(contents as RepositoryContents, current.virtualEntries))
|
|
1648
1668
|
} catch (err) {
|
|
1649
1669
|
if (err instanceof GithubApiError) { sendGithubError(res, err); return }
|
|
1650
1670
|
throw err
|
|
@@ -1669,19 +1689,27 @@ async function handleRepositoryBrowser(res: ServerResponse, deps: ManagedHttpDep
|
|
|
1669
1689
|
// the reader's own access has named the commit, as GitHub would have
|
|
1670
1690
|
// answered them; held by each instance a while, by the database it has, and
|
|
1671
1691
|
// kept in that database for every instance (npm-kept.ts codeTreeStore).
|
|
1692
|
+
// And a directory's listing at a commit (readRepositoryContents), by the
|
|
1693
|
+
// repository's id, the rules it is read by (LISTING_READ), the commit and the
|
|
1694
|
+
// directory, fewer held, each the larger.
|
|
1672
1695
|
const REPOSITORY_TREES_KEPT = 4096
|
|
1696
|
+
const REPOSITORY_LISTINGS_HELD = 256
|
|
1697
|
+
const LISTING_READ = 1
|
|
1673
1698
|
type TreeFiles = Awaited<ReturnType<typeof repositoryTreeFiles>>
|
|
1674
|
-
const repositoryTrees = new WeakMap<ManagedDb, { ids: Lru<string>; files: Lru<TreeFiles> }>()
|
|
1699
|
+
const repositoryTrees = new WeakMap<ManagedDb, { ids: Lru<string>; files: Lru<TreeFiles>; listings: Lru<DirectoryListing> }>()
|
|
1675
1700
|
const treeId = (value: unknown): value is string => typeof value === 'string' && /^[a-f\d]{40}$/u.test(value)
|
|
1676
1701
|
const treeFiles = (value: unknown): value is TreeFiles => value === null || (plainObject(value)
|
|
1677
1702
|
&& Number.isSafeInteger(value['files']) && (value['files'] as number) >= 0 && Number.isSafeInteger(value['bytes']) && (value['bytes'] as number) >= 0)
|
|
1703
|
+
const directoryListing = (value: unknown): value is DirectoryListing => plainObject(value) && Array.isArray(value['entries']) && typeof value['limited'] === 'boolean'
|
|
1678
1704
|
function repositoryTreesOf(deps: ManagedHttpDeps) {
|
|
1679
1705
|
let held = repositoryTrees.get(deps.db)
|
|
1680
|
-
if (!held) repositoryTrees.set(deps.db, held = { ids: lru(REPOSITORY_TREES_KEPT), files: lru(REPOSITORY_TREES_KEPT) })
|
|
1681
|
-
const { ids, files } = held
|
|
1706
|
+
if (!held) repositoryTrees.set(deps.db, held = { ids: lru(REPOSITORY_TREES_KEPT), files: lru(REPOSITORY_TREES_KEPT), listings: lru(REPOSITORY_LISTINGS_HELD) })
|
|
1707
|
+
const { ids, files, listings } = held
|
|
1682
1708
|
return {
|
|
1683
1709
|
id: (key: string, read: () => Promise<string>) => keptTree(ids, codeTreeStore<string>(deps.db, 'trees', deps.config.debug), key, treeId, read),
|
|
1684
1710
|
files: (key: string, read: () => Promise<TreeFiles>) => keptTree(files, codeTreeStore<TreeFiles>(deps.db, 'tree-files', deps.config.debug), key, treeFiles, read),
|
|
1711
|
+
listing: (key: string, read: () => Promise<DirectoryListing>) =>
|
|
1712
|
+
keptTree(listings, codeTreeStore<DirectoryListing>(deps.db, 'listings', deps.config.debug), key, directoryListing, read),
|
|
1685
1713
|
}
|
|
1686
1714
|
}
|
|
1687
1715
|
// What `memory` holds by `key`, else what `store` keeps of it as `kept`
|
|
@@ -1844,7 +1872,8 @@ async function handleRepositoryHistory(res: ServerResponse, deps: ManagedHttpDep
|
|
|
1844
1872
|
// Open pull heads do not depend on the commit history. Start them with
|
|
1845
1873
|
// it, handling failures immediately even if the history fails first.
|
|
1846
1874
|
if (token && path === '' && page === 1) {
|
|
1847
|
-
readingPulls = keptAnswer(deps, `${repoId}\0openPulls`, () => openPullHeads(repo, token), undefined, fresh
|
|
1875
|
+
readingPulls = keptAnswer(deps, `${repoId}\0openPulls`, () => openPullHeads(repo, token), undefined, fresh, fresh ? null : stampOf(deps, repoId, repo, token))
|
|
1876
|
+
.then(answer => answer.value.pulls, quiet(null))
|
|
1848
1877
|
}
|
|
1849
1878
|
let body: object
|
|
1850
1879
|
if (all) {
|
|
@@ -1853,7 +1882,8 @@ async function handleRepositoryHistory(res: ServerResponse, deps: ManagedHttpDep
|
|
|
1853
1882
|
// follow, the deepest page says.
|
|
1854
1883
|
// Asked for afresh, those kept answered as just read: how long till
|
|
1855
1884
|
// they may be read again (`retryIn`), for the client to ask then.
|
|
1856
|
-
const heads = starts ? null : await keptAnswer<AllHeads>(deps, `${repoId}\0heads`, () => repositoryAllHeads(repo, token), undefined, fresh
|
|
1885
|
+
const heads = starts ? null : await keptAnswer<AllHeads>(deps, `${repoId}\0heads`, () => repositoryAllHeads(repo, token), undefined, fresh,
|
|
1886
|
+
fresh ? null : stampOf(deps, repoId, repo, token))
|
|
1857
1887
|
const retryIn = heads && fresh && heads.kept ? heads.at + REPOSITORY_READ_FLOOR_MS - Date.now() : 0
|
|
1858
1888
|
body = starts ? { all: true, page, commits: (await historyFrom(repo, token, starts, pages)).commits, more: page < MAX_HISTORY_PAGE }
|
|
1859
1889
|
: { all: true, ...await repositoryAllHistory(repo, token, heads!.value, pages), ...retryIn > 0 ? { retryIn } : {} }
|
|
@@ -1863,7 +1893,7 @@ async function handleRepositoryHistory(res: ServerResponse, deps: ManagedHttpDep
|
|
|
1863
1893
|
// branches reads and the latest tags, kept for both; without, GitHub's
|
|
1864
1894
|
// first page of each.
|
|
1865
1895
|
const naming = () => token
|
|
1866
|
-
? keptAnswer<AllHeads>(deps, `${repoId}\0heads`, () => repositoryAllHeads(repo, token)).then(({ value }) => ({
|
|
1896
|
+
? keptAnswer<AllHeads>(deps, `${repoId}\0heads`, () => repositoryAllHeads(repo, token), undefined, false, stampOf(deps, repoId, repo, token)).then(({ value }) => ({
|
|
1867
1897
|
branches: value.heads.map(({ name, sha }) => ({ name, sha })), tags: value.tags }))
|
|
1868
1898
|
: keptAnswer(deps, `${repoId}\0refs`, () => repositoryRefTips(repo, token)).then(answer => answer.value)
|
|
1869
1899
|
const tips = page === 1 ? naming() : null
|
|
@@ -2006,8 +2036,9 @@ const REPOSITORY_READ_FLOOR_MS = 10_000
|
|
|
2006
2036
|
// How long a history's answer waits on GitHub looking up the pull requests
|
|
2007
2037
|
// squashed into its commits (handleRepositoryHistory).
|
|
2008
2038
|
const SQUASH_WAIT_MS = 1500
|
|
2009
|
-
// The repositories' reads being read again, behind a kept answer or
|
|
2010
|
-
// for afresh, by key: one at a time, those asking meanwhile
|
|
2039
|
+
// The repositories' reads being read, first or again, behind a kept answer or
|
|
2040
|
+
// as asked for afresh, by key: one at a time, those asking meanwhile
|
|
2041
|
+
// answered by it.
|
|
2011
2042
|
const repositoryRereads = new Map<string, Promise<unknown>>()
|
|
2012
2043
|
// A read GitHub refused, not failed: its credential or the GitHub App's
|
|
2013
2044
|
// permission for it withdrawn (Issues, Pull requests, Checks), the
|
|
@@ -2023,35 +2054,61 @@ const refusedRead = (err: unknown) => err instanceof GithubApiError
|
|
|
2023
2054
|
// deps.waitUntil); `kept` where it was, or where reading it again
|
|
2024
2055
|
// failed as `keepThrough` lets it. One asked for afresh (`fresh`) is read
|
|
2025
2056
|
// now, unless it was read REPOSITORY_READ_FLOOR_MS since, what was kept
|
|
2026
|
-
// answered where that read fails.
|
|
2057
|
+
// answered where that read fails. Those asking for one not kept while it is
|
|
2058
|
+
// read, as a tab's first readers do together, are answered by that read. One GitHub refuses to read again is no
|
|
2027
2059
|
// longer kept: the next to ask is refused as GitHub refused it. `at` is when
|
|
2028
|
-
// what it answers was read.
|
|
2060
|
+
// what it answers was read. With `stamp`, the repository's change stamp
|
|
2061
|
+
// (stampOf), one kept is read again, behind it or afresh, only where the
|
|
2062
|
+
// stamp moved since it was read whole again (the stamp read before it
|
|
2063
|
+
// was), or that was STAMP_TRUSTED_MS ago or longer; else it is kept as read
|
|
2064
|
+
// now, as GitHub would answer it. One read the first time, none's stamp
|
|
2065
|
+
// asked for, is read whole once more before its stamp keeps it.
|
|
2066
|
+
const STAMP_TRUSTED_MS = 60 * 60_000
|
|
2029
2067
|
async function keptAnswer<T extends object>(deps: ManagedHttpDeps, key: string, read: () => Promise<T>,
|
|
2030
|
-
keepThrough: (err: unknown) => boolean = () => false, fresh = false): Promise<{ value: T; kept: boolean; at: number }> {
|
|
2068
|
+
keepThrough: (err: unknown) => boolean = () => false, fresh = false, stamp: Stamp | null = null): Promise<{ value: T; kept: boolean; at: number }> {
|
|
2031
2069
|
const store = repositoryReadStore(deps.db, deps.config.debug)
|
|
2032
2070
|
// One kept longer than a day, not yet removed (pruneKept removes them as
|
|
2033
2071
|
// others are written), is none.
|
|
2034
2072
|
const stored = await store.read(key)
|
|
2035
2073
|
const held = stored && Date.now() - stored.at < REPOSITORY_READ_KEPT_MS ? stored : null
|
|
2036
|
-
//
|
|
2037
|
-
const
|
|
2074
|
+
// The stamp what is kept was read whole at, and when (`read`).
|
|
2075
|
+
const stampKey = `${key}\0stamp`
|
|
2076
|
+
// Read, and kept as read when it was, which it is said read at, with the
|
|
2077
|
+
// stamp read before it (`began`), where one was.
|
|
2078
|
+
const readKept = async (began: string | null = null) => {
|
|
2038
2079
|
try {
|
|
2039
2080
|
const value = await read()
|
|
2040
2081
|
const at = Date.now()
|
|
2041
2082
|
await store.write(key, value, at)
|
|
2042
|
-
|
|
2083
|
+
if (began !== null) await store.write(stampKey, { stamp: began, read: at }, at)
|
|
2084
|
+
return { value, at, kept: false }
|
|
2043
2085
|
} catch (err) {
|
|
2044
2086
|
if (refusedRead(err)) await store.remove(key)
|
|
2045
2087
|
throw err
|
|
2046
2088
|
}
|
|
2047
2089
|
}
|
|
2090
|
+
// Read whole again, or kept as read now where the repository's stamp
|
|
2091
|
+
// hasn't moved since it was.
|
|
2092
|
+
const refresh = async () => {
|
|
2093
|
+
if (held && stamp) {
|
|
2094
|
+
const [told, current] = await Promise.all([store.read(stampKey), stamp().catch(() => null)])
|
|
2095
|
+
const was = told?.value as { stamp?: unknown; read?: unknown } | undefined
|
|
2096
|
+
if (current !== null && was?.stamp === current && typeof was.read === 'number' && Date.now() - was.read < STAMP_TRUSTED_MS) {
|
|
2097
|
+
const at = Date.now()
|
|
2098
|
+
await store.write(key, held.value, at)
|
|
2099
|
+
return { value: held.value as T, at, kept: true }
|
|
2100
|
+
}
|
|
2101
|
+
return readKept(current)
|
|
2102
|
+
}
|
|
2103
|
+
return readKept()
|
|
2104
|
+
}
|
|
2048
2105
|
const reread = () => {
|
|
2049
2106
|
let reading = repositoryRereads.get(key)
|
|
2050
2107
|
if (!reading) {
|
|
2051
|
-
reading =
|
|
2108
|
+
reading = refresh().finally(() => repositoryRereads.delete(key))
|
|
2052
2109
|
repositoryRereads.set(key, reading)
|
|
2053
2110
|
}
|
|
2054
|
-
return reading as Promise<{ value: T; at: number }>
|
|
2111
|
+
return reading as Promise<{ value: T; at: number; kept: boolean }>
|
|
2055
2112
|
}
|
|
2056
2113
|
const age = held ? Date.now() - held.at : Infinity
|
|
2057
2114
|
const now = fresh && age >= REPOSITORY_READ_FLOOR_MS
|
|
@@ -2065,13 +2122,23 @@ async function keptAnswer<T extends object>(deps: ManagedHttpDeps, key: string,
|
|
|
2065
2122
|
return { value: held.value as T, kept: true, at: held.at }
|
|
2066
2123
|
}
|
|
2067
2124
|
try {
|
|
2068
|
-
return
|
|
2125
|
+
return await reread()
|
|
2069
2126
|
} catch (err) {
|
|
2070
2127
|
if (held && ((now && !refusedRead(err)) || keepThrough(err))) return { value: held.value as T, kept: true, at: held.at }
|
|
2071
2128
|
throw err
|
|
2072
2129
|
}
|
|
2073
2130
|
}
|
|
2074
2131
|
|
|
2132
|
+
// Repository `repoId`'s change stamp (repository-stamp.ts), read with `token`
|
|
2133
|
+
// at most every REPOSITORY_READ_FLOOR_MS by every reader together, for its
|
|
2134
|
+
// kept answers to be read again only where it moved (keptAnswer); `also`
|
|
2135
|
+
// what else such an answer moves with, as a period's days do.
|
|
2136
|
+
type Stamp = () => Promise<string>
|
|
2137
|
+
const stampOf = (deps: ManagedHttpDeps, repoId: number, fullName: string, token: string, also = ''): Stamp => async () =>
|
|
2138
|
+
`${(await keptAnswer(deps, `${repoId}\0stamp\0${STAMP_READ}`, () => repositoryStamp(fullName, token), undefined, true)).value.stamp}${also}`
|
|
2139
|
+
// The day it is (UTC), as a period's answer moves with it.
|
|
2140
|
+
const today = () => `\0${Math.floor(Date.now() / (24 * 60 * 60_000))}`
|
|
2141
|
+
|
|
2075
2142
|
// The pages of repository `repoId`'s history kept (historyPageStore), each by
|
|
2076
2143
|
// the commit it starts at (repository-history.ts historyFrom) and the rules
|
|
2077
2144
|
// its co-authors were read by (commit-coauthors.ts COAUTHORS_READ), those
|
|
@@ -2117,37 +2184,103 @@ function commitPulls(deps: ManagedHttpDeps, repoId: number): CommitPulls {
|
|
|
2117
2184
|
}
|
|
2118
2185
|
}
|
|
2119
2186
|
|
|
2120
|
-
//
|
|
2121
|
-
//
|
|
2122
|
-
//
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2187
|
+
// Repository `repoId`'s branches' tips as the Agents tab looks them up
|
|
2188
|
+
// (repository-agents.ts BranchTips), kept by the commit and the rules their
|
|
2189
|
+
// messages' marks were read by (app-agents.ts AGENT_MARKS_READ).
|
|
2190
|
+
function branchTips(deps: ManagedHttpDeps, repoId: number): BranchTips {
|
|
2191
|
+
const store = branchTipStore<BranchTip>(deps.db, deps.config.debug)
|
|
2192
|
+
const keyOf = (sha: string) => `${repoId}\0${AGENT_MARKS_READ}\0${sha}`
|
|
2193
|
+
return {
|
|
2194
|
+
async read(shas) {
|
|
2195
|
+
const kept = await store.readMany(shas.map(keyOf))
|
|
2196
|
+
return new Map(shas.flatMap(sha => { const tip = kept.get(keyOf(sha))?.value; return tip ? [[sha, tip] as const] : [] }))
|
|
2197
|
+
},
|
|
2198
|
+
write: (sha, tip) => store.write(keyOf(sha), tip, Date.now()),
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2127
2201
|
|
|
2128
|
-
//
|
|
2129
|
-
//
|
|
2130
|
-
//
|
|
2131
|
-
//
|
|
2132
|
-
|
|
2202
|
+
// The commits repository `repoId`'s app agents' accounts authored from a
|
|
2203
|
+
// head, as the Agents tab reads them (repository-agents.ts AuthoredCommits),
|
|
2204
|
+
// kept by it and the rules their messages' marks were read by
|
|
2205
|
+
// (AGENT_MARKS_READ).
|
|
2206
|
+
function authoredCommits(deps: ManagedHttpDeps, repoId: number) {
|
|
2207
|
+
const store = authoredCommitStore<AuthoredCommit[]>(deps.db, deps.config.debug)
|
|
2208
|
+
const keyOf = (key: string) => `${repoId}\0${AGENT_MARKS_READ}\0${key}`
|
|
2209
|
+
return {
|
|
2210
|
+
read: async (key: string) => { const kept = (await store.read(keyOf(key)))?.value; return Array.isArray(kept) ? kept : null },
|
|
2211
|
+
write: (key: string, commits: AuthoredCommit[]) => store.write(keyOf(key), commits, Date.now()),
|
|
2212
|
+
}
|
|
2213
|
+
}
|
|
2214
|
+
|
|
2215
|
+
// How long a repository's pull requests or issues kept (keptList) are taken
|
|
2216
|
+
// as they are before those updated since are read: less than their readers
|
|
2217
|
+
// keep their answers (keptAnswer), that one read of them serves them all.
|
|
2218
|
+
const LIST_FRESH_MS = 30_000
|
|
2219
|
+
// Each repository's reads of its pull requests, and of its issues, by them
|
|
2220
|
+
// and its id: one at a time, each after those asked before it.
|
|
2221
|
+
const listReads = new Map<string, Promise<unknown>>()
|
|
2222
|
+
|
|
2223
|
+
// A repository's pull requests or issues, as kept in `store` by `key`
|
|
2224
|
+
// (repository-pulls.ts readList, `read`), read on where they were read
|
|
2225
|
+
// LIST_FRESH_MS ago or longer, or not as far back as `need`: as far back as
|
|
2226
|
+
// it, for the Pulse; for the Agents tab (null), as far back as they were
|
|
2227
|
+
// read, those updated since read (a read not finished, on), else their
|
|
2228
|
+
// first page. Not to be read further back (`further` false): those kept,
|
|
2229
|
+
// none where none are, where they are not read as far as `need`, or a read
|
|
2230
|
+
// of them is not finished, those updated since read alone. One read of a
|
|
2133
2231
|
// repository's at a time, the rest after it, from what it kept.
|
|
2134
|
-
function
|
|
2135
|
-
|
|
2136
|
-
const
|
|
2137
|
-
const reading = (pullsReads.get(repoId) ?? Promise.resolve()).catch(() => {}).then(async () => {
|
|
2232
|
+
function keptList<L extends Listed<unknown>>(store: KeptStore<L>, key: string, read: (since: number, kept: L | null, need: number) => Promise<L>,
|
|
2233
|
+
need: number | null, further = true): Promise<L | null> {
|
|
2234
|
+
const reading = (listReads.get(key) ?? Promise.resolve()).catch(() => {}).then(async () => {
|
|
2138
2235
|
const kept = await store.read(key)
|
|
2139
2236
|
const list = kept?.value ?? null
|
|
2140
|
-
if (list && Date.now() - kept!.at <
|
|
2237
|
+
if (list && Date.now() - kept!.at < LIST_FRESH_MS && (need === null || pullsReadTo(list) <= need)) return list
|
|
2238
|
+
if (!further && need !== null && (!list || list.next || pullsReadTo(list) > need)) return list
|
|
2141
2239
|
const since = pulseSince(52, Date.now())
|
|
2142
|
-
const
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
return read
|
|
2240
|
+
const listed = await read(since, list, need ?? (list ? list.next ? since : list.reach ?? since : Date.now()))
|
|
2241
|
+
await store.write(key, listed, Date.now())
|
|
2242
|
+
return listed
|
|
2146
2243
|
})
|
|
2147
|
-
|
|
2148
|
-
void reading.catch(() => {}).finally(() => { if (
|
|
2244
|
+
listReads.set(key, reading)
|
|
2245
|
+
void reading.catch(() => {}).finally(() => { if (listReads.get(key) === reading) listReads.delete(key) })
|
|
2149
2246
|
return reading
|
|
2150
2247
|
}
|
|
2248
|
+
// Repository `repoId`'s pull requests (keptList), kept by how agents are
|
|
2249
|
+
// told too; those kept alone, not read further back (`further` false).
|
|
2250
|
+
function keptPulls(deps: ManagedHttpDeps, repoId: number, fullName: string, token: string, need: number | null): Promise<PullList>
|
|
2251
|
+
function keptPulls(deps: ManagedHttpDeps, repoId: number, fullName: string, token: string, need: number, further: false): Promise<PullList | null>
|
|
2252
|
+
function keptPulls(deps: ManagedHttpDeps, repoId: number, fullName: string, token: string, need: number | null, further = true) {
|
|
2253
|
+
return keptList(pullListStore<PullList>(deps.db, deps.config.debug), `${repoId}\0${PULLS_READ_VERSION}\0${AGENTS_READ_VERSION}\0${PULSE_YEAR_VERSION}`,
|
|
2254
|
+
(since, kept, asked) => readPulls(fullName, token, since, kept, globalThis.fetch, asked), need, further)
|
|
2255
|
+
}
|
|
2256
|
+
// Repository `repoId`'s issues (keptList).
|
|
2257
|
+
const keptIssues = (deps: ManagedHttpDeps, repoId: number, fullName: string, token: string, need: number | null) =>
|
|
2258
|
+
keptList(issueListStore<IssueList>(deps.db, deps.config.debug), `${repoId}\0${ISSUES_LIST_READ_VERSION}`,
|
|
2259
|
+
(since, kept, asked) => readIssues(fullName, token, since, kept, globalThis.fetch, asked), need) as Promise<IssueList>
|
|
2260
|
+
|
|
2261
|
+
// What is kept by `key` (keptAnswer), read now where it was read
|
|
2262
|
+
// REPOSITORY_READ_FRESH_MS ago or longer, not behind: for an answer that is
|
|
2263
|
+
// kept as fresh itself, as told of it (the Agents tab's).
|
|
2264
|
+
async function currentAnswer<T extends object>(deps: ManagedHttpDeps, key: string, read: () => Promise<T>, stamp: Stamp | null = null) {
|
|
2265
|
+
const stored = await repositoryReadStore(deps.db, deps.config.debug).read(key)
|
|
2266
|
+
return keptAnswer(deps, key, read, undefined, !stored || Date.now() - stored.at >= REPOSITORY_READ_FRESH_MS, stamp)
|
|
2267
|
+
}
|
|
2268
|
+
|
|
2269
|
+
// Repository `repoId`'s default branch's head, as its tabs read it: as the
|
|
2270
|
+
// Commits tab's branches last told it (`heads`), where they were read
|
|
2271
|
+
// fresh, else read alone and kept (`pulse-head`), read now where that is no
|
|
2272
|
+
// longer fresh as asked (`current`); whether it was kept (`kept`), GitHub
|
|
2273
|
+
// asked for none of it.
|
|
2274
|
+
async function defaultTip(deps: ManagedHttpDeps, repoId: number, fullName: string, token: string, current = false): Promise<{ sha: string | null; kept: boolean }> {
|
|
2275
|
+
const heads = await repositoryReadStore(deps.db, deps.config.debug).read(`${repoId}\0heads`)
|
|
2276
|
+
const all = heads && Date.now() - heads.at < REPOSITORY_READ_FRESH_MS ? heads.value as AllHeads : null
|
|
2277
|
+
const tip = all?.heads.find(head => head.name === all.defaultBranch)?.sha
|
|
2278
|
+
if (typeof tip === 'string' && /^[a-f\d]{40}$/u.test(tip)) return { sha: tip, kept: true }
|
|
2279
|
+
const head = async () => ({ sha: await defaultHead(fullName, token) })
|
|
2280
|
+
const stamp = stampOf(deps, repoId, fullName, token)
|
|
2281
|
+
const read = await (current ? currentAnswer(deps, `${repoId}\0pulse-head`, head, stamp) : keptAnswer(deps, `${repoId}\0pulse-head`, head, undefined, false, stamp))
|
|
2282
|
+
return { sha: read.value.sha, kept: read.kept }
|
|
2283
|
+
}
|
|
2151
2284
|
|
|
2152
2285
|
// What the Pulse keeps of repository `repoId`'s pull requests merged, as
|
|
2153
2286
|
// it tells by them whose its year's commits are (pulseCommits, agentMerges).
|
|
@@ -2195,10 +2328,10 @@ function historySquashes(deps: ManagedHttpDeps, repoId: number): Squashes {
|
|
|
2195
2328
|
// `readAt`, when it was read from GitHub, one kept as old as it was kept, so
|
|
2196
2329
|
// that a reader holding two reads of the same can tell the later.
|
|
2197
2330
|
async function readWholeRepository(res: ServerResponse, deps: ManagedHttpDeps, cookie: string | undefined, repoId: number,
|
|
2198
|
-
read: (fullName: string, token: string) => Promise<object>, key: string | null = null, readAt = false): Promise<void> {
|
|
2331
|
+
read: (fullName: string, token: string) => Promise<object>, key: string | null = null, readAt = false, stamped = false): Promise<void> {
|
|
2199
2332
|
await answerWholeRepository(res, deps, cookie, repoId, async (fullName, token) => {
|
|
2200
2333
|
const { value, kept, at } = key === null ? { value: await read(fullName, token), kept: false, at: Date.now() }
|
|
2201
|
-
: await keptAnswer(deps, `${repoId}\0${key}`, () => read(fullName, token))
|
|
2334
|
+
: await keptAnswer(deps, `${repoId}\0${key}`, () => read(fullName, token), undefined, false, stamped ? stampOf(deps, repoId, fullName, token) : null)
|
|
2202
2335
|
return { value: readAt ? { ...value, readAt: at } : value, kept }
|
|
2203
2336
|
})
|
|
2204
2337
|
}
|
|
@@ -2264,6 +2397,22 @@ async function readWholeRepositoryOf<T>(deps: ManagedHttpDeps, access: WholeAcce
|
|
|
2264
2397
|
}
|
|
2265
2398
|
}
|
|
2266
2399
|
|
|
2400
|
+
// The patch `patch` names (npm-patch.ts), read of the whole of its repository
|
|
2401
|
+
// as readWholeRepositoryOf reads it, with its reader's own GitHub access, and
|
|
2402
|
+
// held to its hash: as kept, else with that access checked again after; else
|
|
2403
|
+
// why it was refused. Its failing to read or to apply throws.
|
|
2404
|
+
async function npmPatchOf(deps: ManagedHttpDeps, cookie: string | undefined, patch: NpmPatch): Promise<{ files: PatchedFile[] } | Refusal> {
|
|
2405
|
+
const access = await wholeRepositoryAccess(deps, cookie, patch.repoId)
|
|
2406
|
+
if ('error' in access) return access
|
|
2407
|
+
const read = await readWholeRepositoryOf(deps, access, async (fullName, token) => {
|
|
2408
|
+
const asked: GithubAsked = { github: false }
|
|
2409
|
+
return { value: await readPatch(fullName, token, patch, asked), kept: !asked.github }
|
|
2410
|
+
}, () => wholeRepositoryAccess(deps, cookie, patch.repoId))
|
|
2411
|
+
return 'error' in read ? read : { files: read.value }
|
|
2412
|
+
}
|
|
2413
|
+
// The patch a request's fields name, as npm-patch.ts requestedPatch takes them.
|
|
2414
|
+
const patchFields = (field: (key: string) => unknown) => requestedPatch({ hash: field('patchHash'), repoId: field('patchRepoId'), commit: field('patchCommit'), path: field('patchPath') })
|
|
2415
|
+
|
|
2267
2416
|
// GET /api/admin/repositories/issues: what the code viewer's Issues and Pull
|
|
2268
2417
|
// requests tabs show (repository-issues.ts): `kind`'s open or closed
|
|
2269
2418
|
// (`state`), a page at a time, on from the cursor `after` names. They are
|
|
@@ -2279,7 +2428,10 @@ async function handleRepositoryIssues(res: ServerResponse, deps: ManagedHttpDeps
|
|
|
2279
2428
|
|| (after !== null && !ISSUES_CURSOR.test(after))) { sendJson(res, 400, { error: 'bad-request' }); return }
|
|
2280
2429
|
await answerWholeRepository(res, deps, cookie, repoId, async (fullName, token) => {
|
|
2281
2430
|
const { value: { commits: _commits, ...page }, kept, at } = await keptAnswer(deps, `${repoId}\0issues\0${ISSUES_READ_VERSION}\0${kind}\0${state}\0${after ?? ''}`,
|
|
2282
|
-
async () => ({ kind, state, ...await repositoryIssues(fullName, token, kind, state, after) })
|
|
2431
|
+
async () => ({ kind, state, ...await repositoryIssues(fullName, token, kind, state, after) }), undefined, false,
|
|
2432
|
+
// An issue's tile tells nothing its update time doesn't move with; a
|
|
2433
|
+
// pull request's, its checks.
|
|
2434
|
+
kind === 'issues' ? stampOf(deps, repoId, fullName, token) : null)
|
|
2283
2435
|
return { value: { ...page, readAt: at }, kept }
|
|
2284
2436
|
})
|
|
2285
2437
|
}
|
|
@@ -2298,8 +2450,11 @@ async function handleRepositoryAgents(res: ServerResponse, deps: ManagedHttpDeps
|
|
|
2298
2450
|
sendJson(res, 400, { error: 'bad-request' }); return
|
|
2299
2451
|
}
|
|
2300
2452
|
await readWholeRepository(res, deps, cookie, repoId, (fullName, token) => repositoryAgents(fullName, token, cursors, historyPages(deps, repoId, true, true),
|
|
2301
|
-
commitPulls(deps, repoId), () => keptPulls(deps, repoId, fullName, token, null)),
|
|
2302
|
-
|
|
2453
|
+
commitPulls(deps, repoId), { pulls: () => keptPulls(deps, repoId, fullName, token, null), issues: () => keptIssues(deps, repoId, fullName, token, null),
|
|
2454
|
+
head: async () => (await defaultTip(deps, repoId, fullName, token, true)).sha,
|
|
2455
|
+
heads: async () => (await currentAnswer<AllHeads>(deps, `${repoId}\0heads`, () => repositoryAllHeads(fullName, token), stampOf(deps, repoId, fullName, token))).value,
|
|
2456
|
+
tips: branchTips(deps, repoId), authored: authoredCommits(deps, repoId) }),
|
|
2457
|
+
`agents\0${AGENTS_READ_VERSION}\0${cursors.pulls ?? ''}\0${cursors.commits ?? ''}`, false, true)
|
|
2303
2458
|
}
|
|
2304
2459
|
|
|
2305
2460
|
// GET /api/admin/repositories/item: the issue or pull request numbered
|
|
@@ -2338,13 +2493,19 @@ const PULLS_AT_ONCE = 3
|
|
|
2338
2493
|
async function handleRepositoryPullDependencies(res: ServerResponse, deps: ManagedHttpDeps, cookie: string | undefined, query: URLSearchParams): Promise<void> {
|
|
2339
2494
|
const repoId = Number(query.get('repoId'))
|
|
2340
2495
|
const review = query.get('review') === '1'
|
|
2496
|
+
// As an open one's tile on the Pull requests tab asks (`listed=1`): each
|
|
2497
|
+
// by the commits the page it is of names, where one kept names them
|
|
2498
|
+
// (keptPullCommits); else by its number, kept a while, as a popup's.
|
|
2499
|
+
const tiles = query.get('listed') === '1'
|
|
2341
2500
|
const listed = query.get('numbers')
|
|
2342
2501
|
const numbers = listed === null ? [itemNumber(query)] : listed.split(',').map(text => itemNumber(new URLSearchParams({ number: text })))
|
|
2343
2502
|
if (!Number.isSafeInteger(repoId) || repoId <= 0 || numbers.length > MAX_ASKED_PULLS || numbers.some(number => number === null) || new Set(numbers).size < numbers.length
|
|
2344
|
-
|| !['1', null].includes(query.get('review'))) { sendJson(res, 400, { error: 'bad-request' }); return }
|
|
2503
|
+
|| !['1', null].includes(query.get('review')) || !['1', null].includes(query.get('listed'))) { sendJson(res, 400, { error: 'bad-request' }); return }
|
|
2345
2504
|
await answerWholeRepository(res, deps, cookie, repoId, async (fullName, token, user) => {
|
|
2346
|
-
|
|
2347
|
-
|
|
2505
|
+
const commits = tiles ? await keptPullCommits(deps, repoId, numbers as number[]) : new Map<number, PullCommits>()
|
|
2506
|
+
if (listed === null) return pullDependencyAnswer(deps, fullName, token, user, repoId, numbers[0]!, review, commits.get(numbers[0]!))
|
|
2507
|
+
const answers = await eachOf(numbers as number[], PULLS_AT_ONCE, number => pullDependencyAnswer(deps, fullName, token, user, repoId, number, review, commits.get(number))
|
|
2508
|
+
.catch((err: unknown) => {
|
|
2348
2509
|
// GitHub asked, though it failed: its access asked again after.
|
|
2349
2510
|
if (err instanceof GithubApiError) return { value: { number, error: err.message }, kept: false }
|
|
2350
2511
|
throw err
|
|
@@ -2366,7 +2527,7 @@ async function pullDependencyAnswer(deps: ManagedHttpDeps, fullName: string, tok
|
|
|
2366
2527
|
const read = await pullDependenciesAt(fullName, token, number, kept)
|
|
2367
2528
|
reading.told = read.told
|
|
2368
2529
|
return { commits: read.commits }
|
|
2369
|
-
})
|
|
2530
|
+
}, undefined, false, stampOf(deps, repoId, fullName, token))
|
|
2370
2531
|
// What its commits tell, as read, else as kept by them, else, no longer
|
|
2371
2532
|
// kept, read again: by those given, without asking GitHub for them.
|
|
2372
2533
|
const held = reading.told ?? (typeof answered.value.commits === 'string' ? await kept.read(answered.value.commits) : null)
|
|
@@ -2416,8 +2577,8 @@ async function handleRepositoryBranchDependencies(res: ServerResponse, deps: Man
|
|
|
2416
2577
|
// setting as read already (`known`), where it was.
|
|
2417
2578
|
async function branchDependencyAnswer(deps: ManagedHttpDeps, fullName: string, token: string, user: StoredUser, repoId: number,
|
|
2418
2579
|
known?: RepoDependencyReview | null): Promise<{ value: object; kept: boolean }> {
|
|
2419
|
-
const headRead = await
|
|
2420
|
-
const head = headRead.
|
|
2580
|
+
const headRead = await defaultTip(deps, repoId, fullName, token)
|
|
2581
|
+
const head = headRead.sha
|
|
2421
2582
|
if (!head) return { value: { head: null, review: null }, kept: false }
|
|
2422
2583
|
const asked: GithubAsked = { github: !headRead.kept }
|
|
2423
2584
|
// Those its review holds to alone read, as many as are read of them.
|
|
@@ -2447,7 +2608,7 @@ async function handleRepositoryLockfiles(res: ServerResponse, deps: ManagedHttpD
|
|
|
2447
2608
|
const repoId = Number(query.get('repoId'))
|
|
2448
2609
|
if (!Number.isSafeInteger(repoId) || repoId <= 0) { sendJson(res, 400, { error: 'bad-request' }); return }
|
|
2449
2610
|
await answerWholeRepository(res, deps, cookie, repoId, async (fullName, token) => {
|
|
2450
|
-
const head = (await
|
|
2611
|
+
const head = (await defaultTip(deps, repoId, fullName, token)).sha
|
|
2451
2612
|
return { value: { head, lockfiles: head ? (await lockfilesAt(fullName, token, head, lockfileKept(deps, repoId))).map(({ path, format }) => ({ path, format })) : [] }, kept: false }
|
|
2452
2613
|
})
|
|
2453
2614
|
}
|
|
@@ -2585,36 +2746,63 @@ async function teamOpenPulls(deps: ManagedHttpDeps, repoId: number, fullName: st
|
|
|
2585
2746
|
let after: string | null = null, kept = true
|
|
2586
2747
|
for (let page = 0; page < TEAM_PULL_PAGES; page++) {
|
|
2587
2748
|
const cursor = after
|
|
2588
|
-
const answer = await keptAnswer(deps,
|
|
2589
|
-
async () => ({ kind: 'pulls', state: 'open', ...await repositoryIssues(fullName, token, 'pulls', 'open', cursor) }))
|
|
2749
|
+
const answer = await keptAnswer(deps, openPullsKey(repoId, cursor), async () => ({ kind: 'pulls', state: 'open', ...await repositoryIssues(fullName, token, 'pulls', 'open', cursor) }))
|
|
2590
2750
|
kept &&= answer.kept
|
|
2591
|
-
const value = answer.value as
|
|
2751
|
+
const value = answer.value as OpenPullsPage
|
|
2592
2752
|
items.push(...(Array.isArray(value.items) ? value.items : []).filter(item => Number.isSafeInteger(item?.number) && item.number > 0))
|
|
2593
|
-
|
|
2594
|
-
const { number, base, head } = pull ?? {}
|
|
2595
|
-
if (Number.isSafeInteger(number) && typeof base === 'string' && OBJECT_ID.test(base) && typeof head === 'string' && OBJECT_ID.test(head)) {
|
|
2596
|
-
commits.set(number as number, { base, head })
|
|
2597
|
-
}
|
|
2598
|
-
}
|
|
2753
|
+
pageCommits(value, commits)
|
|
2599
2754
|
if (value.more !== true || typeof value.after !== 'string') return { items, commits, more: false, kept }
|
|
2600
2755
|
after = value.after
|
|
2601
2756
|
}
|
|
2602
2757
|
return { items, commits, more: true, kept }
|
|
2603
2758
|
}
|
|
2759
|
+
type OpenPullsPage = { items?: unknown; more?: unknown; after?: unknown; commits?: unknown }
|
|
2760
|
+
const openPullsKey = (repoId: number, cursor: string | null) => `${repoId}\0issues\0${ISSUES_READ_VERSION}\0pulls\0open\0${cursor ?? ''}`
|
|
2761
|
+
// Each open pull request's base's and head's commits a page of them names,
|
|
2762
|
+
// into `commits` by its number.
|
|
2763
|
+
function pageCommits(page: OpenPullsPage, commits: Map<number, PullCommits>) {
|
|
2764
|
+
for (const pull of Array.isArray(page.commits) ? page.commits as { number?: unknown; base?: unknown; head?: unknown }[] : []) {
|
|
2765
|
+
const { number, base, head } = pull ?? {}
|
|
2766
|
+
if (Number.isSafeInteger(number) && typeof base === 'string' && OBJECT_ID.test(base) && typeof head === 'string' && OBJECT_ID.test(head)) {
|
|
2767
|
+
commits.set(number as number, { base, head })
|
|
2768
|
+
}
|
|
2769
|
+
}
|
|
2770
|
+
}
|
|
2771
|
+
|
|
2772
|
+
// Open pull requests `numbers`' base's and head's commits, as the pages of
|
|
2773
|
+
// them the Pull requests tab read were kept with them (GET issues), as many
|
|
2774
|
+
// of those as TEAM_PULL_PAGES on from its first, while all are kept: GitHub
|
|
2775
|
+
// not asked, nor pages read where none is kept. A tile of a page kept, even
|
|
2776
|
+
// one read again behind it, is of the commits it names.
|
|
2777
|
+
async function keptPullCommits(deps: ManagedHttpDeps, repoId: number, numbers: number[]) {
|
|
2778
|
+
const store = repositoryReadStore(deps.db, deps.config.debug)
|
|
2779
|
+
const commits = new Map<number, PullCommits>()
|
|
2780
|
+
let after: string | null = null
|
|
2781
|
+
for (let page = 0; page < TEAM_PULL_PAGES && numbers.some(number => !commits.has(number)); page++) {
|
|
2782
|
+
const stored = await store.read(openPullsKey(repoId, after))
|
|
2783
|
+
if (!stored || Date.now() - stored.at >= REPOSITORY_READ_KEPT_MS) break
|
|
2784
|
+
const value = stored.value as OpenPullsPage
|
|
2785
|
+
pageCommits(value, commits)
|
|
2786
|
+
if (value.more !== true || typeof value.after !== 'string') break
|
|
2787
|
+
after = value.after
|
|
2788
|
+
}
|
|
2789
|
+
return commits
|
|
2790
|
+
}
|
|
2604
2791
|
|
|
2605
2792
|
// The lines the versions wanting reviews of `answers` change, as their rows
|
|
2606
|
-
// show them (npm-delta.js deltaBase), where they are
|
|
2607
|
-
// public versions alone, those npm answered for in a day
|
|
2608
|
-
// their integrities those npmDelta keys them by. `[name,
|
|
2609
|
-
// delta]` each, none counted here.
|
|
2793
|
+
// show them (npm-delta.js deltaBase), a patched one's none, where they are
|
|
2794
|
+
// counted already: of public versions alone, those npm answered for in a day
|
|
2795
|
+
// (npmVersionStore), their integrities those npmDelta keys them by. `[name,
|
|
2796
|
+
// version, base, delta]` each, none counted here.
|
|
2610
2797
|
async function teamDeltas(deps: ManagedHttpDeps, answers: object[]) {
|
|
2611
2798
|
const asked = new Map<string, [string, string, string]>()
|
|
2612
2799
|
for (const answer of answers as { lockfiles?: { added?: unknown; unreviewed?: unknown }[]; review?: { required?: number } }[]) {
|
|
2613
2800
|
const required = answer.review?.required
|
|
2614
2801
|
if (!Number.isSafeInteger(required)) continue
|
|
2615
2802
|
for (const pkg of (answer.lockfiles ?? []).flatMap(dir => [...Array.isArray(dir.added) ? dir.added : [], ...Array.isArray(dir.unreviewed) ? dir.unreviewed : []]) as
|
|
2616
|
-
{ name?: unknown; version?: unknown; source?: unknown; was?: unknown; reviews?: unknown; rejected?: unknown; baseline?: unknown; reviewed?: unknown; unsupported?: unknown
|
|
2617
|
-
|
|
2803
|
+
{ name?: unknown; version?: unknown; source?: unknown; was?: unknown; reviews?: unknown; rejected?: unknown; baseline?: unknown; reviewed?: unknown; unsupported?: unknown
|
|
2804
|
+
patch?: unknown }[]) {
|
|
2805
|
+
if (pkg.source !== 'registry' || pkg.patch || typeof pkg.name !== 'string' || typeof pkg.version !== 'string' || !isExactVersion(pkg.version)) continue
|
|
2618
2806
|
if (pkg.unsupported || pkg.baseline || pkg.reviewed || typeof pkg.reviews !== 'number' || (pkg.reviews >= required! && !pkg.rejected)) continue
|
|
2619
2807
|
const was = Array.isArray(pkg.was) ? pkg.was.find(prior => typeof prior === 'string' && prior !== pkg.version) as string | undefined : undefined
|
|
2620
2808
|
asked.set(`${pkg.name}\0${pkg.version}\0${was ?? '(empty)'}`, [pkg.name, pkg.version, was ?? '(empty)'])
|
|
@@ -2718,15 +2906,55 @@ async function handleRepositoryPulse(res: ServerResponse, deps: ManagedHttpDeps,
|
|
|
2718
2906
|
const weeks = Number(query.get('weeks') ?? '4')
|
|
2719
2907
|
if (!Number.isSafeInteger(repoId) || repoId <= 0 || !PULSE_WEEKS.includes(weeks)) { sendJson(res, 400, { error: 'bad-request' }); return }
|
|
2720
2908
|
await answerWholeRepository(res, deps, cookie, repoId, async (fullName, token) => {
|
|
2721
|
-
const
|
|
2722
|
-
|
|
2723
|
-
|
|
2909
|
+
const year = pulseYearRead(deps, repoId, fullName, token, weeks)
|
|
2910
|
+
const [period, counted] = await Promise.all([
|
|
2911
|
+
keptAnswer(deps, `${repoId}\0pulse\0${weeks}`, () => pulseActivity(deps, repoId, fullName, token, weeks), undefined, false, stampOf(deps, repoId, fullName, token, today())),
|
|
2912
|
+
Promise.race([year.reading, delay(PULSE_YEAR_WAIT_MS, null, { ref: false })]),
|
|
2724
2913
|
])
|
|
2725
|
-
|
|
2726
|
-
|
|
2914
|
+
const told = counted ?? { stats: await lastPulseYear(deps, repoId, pulseSince(52, Date.now())), reading: year.progress.commits, pulls: null, pending: false,
|
|
2915
|
+
kept: false }
|
|
2916
|
+
return { value: { weeks, ...period.value, stats: told.stats && periodStats(told.stats, weeks), reading: told.reading, pulls: told.pulls,
|
|
2917
|
+
pending: told.pending, ...counted ? {} : { behind: true } }, kept: period.kept && told.kept }
|
|
2727
2918
|
})
|
|
2728
2919
|
}
|
|
2729
2920
|
|
|
2921
|
+
// How long a Pulse answer waits on its year (pulseCommits) before it is
|
|
2922
|
+
// sent without it: the period's pull requests and issues told, and how many
|
|
2923
|
+
// of the year's commits are read so far (`reading`, beside the year last
|
|
2924
|
+
// counted), the year read on behind it (`behind`, for the client to ask
|
|
2925
|
+
// again till it is, however long a page of it takes to read); those
|
|
2926
|
+
// asking meanwhile, by the database, the repository and the period, waiting
|
|
2927
|
+
// on that read, not one of their own, unless it began PULSE_YEAR_HELD_MS ago
|
|
2928
|
+
// or longer.
|
|
2929
|
+
const PULSE_YEAR_WAIT_MS = 1500
|
|
2930
|
+
const PULSE_YEAR_HELD_MS = 60_000
|
|
2931
|
+
type PulseYear = Awaited<ReturnType<typeof pulseCommits>>
|
|
2932
|
+
type PulseYearRead = { reading: Promise<PulseYear>; progress: { commits: number }; began: number }
|
|
2933
|
+
const pulseYearReads = new WeakMap<ManagedDb, Map<string, PulseYearRead>>()
|
|
2934
|
+
function pulseYearRead(deps: ManagedHttpDeps, repoId: number, fullName: string, token: string, weeks: number) {
|
|
2935
|
+
let reads = pulseYearReads.get(deps.db)
|
|
2936
|
+
if (!reads) pulseYearReads.set(deps.db, reads = new Map())
|
|
2937
|
+
const key = `${repoId}\0${weeks}`
|
|
2938
|
+
const held = reads.get(key)
|
|
2939
|
+
if (held && Date.now() - held.began < PULSE_YEAR_HELD_MS) return held
|
|
2940
|
+
const progress = { commits: 0 }
|
|
2941
|
+
const year = { reading: pulseCommits(deps, repoId, fullName, token, weeks, progress), progress, began: Date.now() }
|
|
2942
|
+
reads.set(key, year)
|
|
2943
|
+
const settled = year.reading.catch((err: unknown) => { console.warn('managed: pulse year read failed:', err) })
|
|
2944
|
+
.finally(() => { if (reads.get(key) === year) reads.delete(key) })
|
|
2945
|
+
deps.track(settled)
|
|
2946
|
+
deps.waitUntil?.(settled)
|
|
2947
|
+
return year
|
|
2948
|
+
}
|
|
2949
|
+
|
|
2950
|
+
// The year last counted of repository `repoId` (pulseCommits), where it is
|
|
2951
|
+
// no more than PULSE_LAST_MS behind the year from `first`; else none.
|
|
2952
|
+
const pulseLastKey = (repoId: number) => `${repoId}\0pulse-year-last`
|
|
2953
|
+
async function lastPulseYear(deps: ManagedHttpDeps, repoId: number, first: number) {
|
|
2954
|
+
const kept = (await repositoryReadStore(deps.db, deps.config.debug).read(pulseLastKey(repoId)))?.value as { first: number; stats: PulseStats } | undefined
|
|
2955
|
+
return kept && first - kept.first <= PULSE_LAST_MS ? kept.stats : null
|
|
2956
|
+
}
|
|
2957
|
+
|
|
2730
2958
|
// The default branch's year (repository-pulse.ts commitStats), counted from
|
|
2731
2959
|
// its commits as the Commits tab keeps them, with their lines
|
|
2732
2960
|
// (yearCommits): kept by the head it was read from (repositoryReadStore),
|
|
@@ -2761,6 +2989,25 @@ const pulseDay = (value: string | null) => {
|
|
|
2761
2989
|
return Number.isSafeInteger(at) ? at : null
|
|
2762
2990
|
}
|
|
2763
2991
|
|
|
2992
|
+
// The pull requests and issues of a period of `weeks` weeks, narrowed as
|
|
2993
|
+
// asked (`narrowing`), as the Pulse shows them: of those kept
|
|
2994
|
+
// (listed-activity.ts listedActivity), where they are read whole as far
|
|
2995
|
+
// back as the period before it starts, within the year they are read for:
|
|
2996
|
+
// the issues read on as far first; the pull requests as the year's commits
|
|
2997
|
+
// read them (pulseCommits), never further for this; else searched for
|
|
2998
|
+
// (repositoryActivity), as while they are read.
|
|
2999
|
+
async function pulseActivity(deps: ManagedHttpDeps, repoId: number, fullName: string, token: string, weeks: number, narrowing: PulseNarrowing | null = null) {
|
|
3000
|
+
const now = Date.now()
|
|
3001
|
+
// As far back as the period before it starts, and its narrowing's first day.
|
|
3002
|
+
const need = Math.min(pulseSince(2 * weeks, now), narrowing?.from ?? Number.POSITIVE_INFINITY)
|
|
3003
|
+
if (need >= pulseSince(52, now)) {
|
|
3004
|
+
const [pulls, issues] = await Promise.all([keptPulls(deps, repoId, fullName, token, need, false), keptIssues(deps, repoId, fullName, token, need)])
|
|
3005
|
+
const whole = (list: Listed<unknown> | null): list is Listed<unknown> => list !== null && !list.next && pullsReadTo(list) <= need
|
|
3006
|
+
if (whole(pulls) && whole(issues)) return listedActivity(pulls.rows, issues.rows, weeks, now, narrowing)
|
|
3007
|
+
}
|
|
3008
|
+
return repositoryActivity(fullName, token, weeks, now, globalThis.fetch, narrowing)
|
|
3009
|
+
}
|
|
3010
|
+
|
|
2764
3011
|
// GET /api/admin/repositories/pulse/items: the Pulse tab's pull requests and
|
|
2765
3012
|
// issues narrowed to a picked day or week (`from` to `to`, else on from
|
|
2766
3013
|
// `from`) and the GitHub accounts a picked contributor committed as
|
|
@@ -2777,38 +3024,37 @@ async function handleRepositoryPulseItems(res: ServerResponse, deps: ManagedHttp
|
|
|
2777
3024
|
await answerWholeRepository(res, deps, cookie, repoId, async (fullName, token) => {
|
|
2778
3025
|
const narrowing = { from, to, authors }
|
|
2779
3026
|
const items = await keptAnswer(deps, `${repoId}\0pulse-items\0${weeks}\0${from}\0${to ?? ''}\0${narrowing.authors.join(',')}`,
|
|
2780
|
-
() =>
|
|
3027
|
+
() => pulseActivity(deps, repoId, fullName, token, weeks, narrowing), undefined, false, stampOf(deps, repoId, fullName, token, today()))
|
|
2781
3028
|
return { value: { weeks, ...items.value }, kept: items.kept }
|
|
2782
3029
|
})
|
|
2783
3030
|
}
|
|
2784
3031
|
|
|
2785
|
-
async function pulseCommits(deps: ManagedHttpDeps, repoId: number, fullName: string, token: string, weeks = 52) {
|
|
3032
|
+
async function pulseCommits(deps: ManagedHttpDeps, repoId: number, fullName: string, token: string, weeks = 52, progress: { commits: number } | null = null) {
|
|
2786
3033
|
const now = Date.now()
|
|
2787
3034
|
const first = pulseSince(52, now)
|
|
2788
3035
|
// As far back as a period of `weeks` weeks needs the year read whole: its
|
|
2789
3036
|
// weeks and as many before them, which it is shown beside; the year's,
|
|
2790
3037
|
// all of it.
|
|
2791
3038
|
const need = Math.max(first, pulseSince(2 * weeks, now))
|
|
2792
|
-
const head = await
|
|
2793
|
-
if (!head.
|
|
3039
|
+
const head = await defaultTip(deps, repoId, fullName, token)
|
|
3040
|
+
if (!head.sha) return { stats: commitStats([], first), reading: null, pulls: null, pending: false, kept: head.kept }
|
|
2794
3041
|
const store = repositoryReadStore(deps.db, deps.config.debug)
|
|
2795
|
-
const key = `${repoId}\0pulse-year\0${PULSE_YEAR_VERSION}\0${head.
|
|
2796
|
-
const lastKey =
|
|
3042
|
+
const key = `${repoId}\0pulse-year\0${PULSE_YEAR_VERSION}\0${head.sha}\0${first}`
|
|
3043
|
+
const lastKey = pulseLastKey(repoId)
|
|
2797
3044
|
const mergesKey = pulseMergesKey(repoId)
|
|
2798
|
-
const last =
|
|
2799
|
-
|
|
2800
|
-
return kept && first - kept.first <= PULSE_LAST_MS ? kept.stats : null
|
|
2801
|
-
}
|
|
2802
|
-
const sha = head.value.sha
|
|
3045
|
+
const last = () => lastPulseYear(deps, repoId, first)
|
|
3046
|
+
const sha = head.sha
|
|
2803
3047
|
// What tells whose the year's commits are, kept as read: the agents' pull
|
|
2804
3048
|
// requests merged (agentMerges), as the pull requests kept tell, read as
|
|
2805
3049
|
// far back as `whole` (keptPulls), and the commits that release the
|
|
2806
|
-
// repository (`releases`, release-commits.ts releaseCommits)
|
|
3050
|
+
// repository since then (`releases`, release-commits.ts releaseCommits),
|
|
3051
|
+
// as far back as what is counted whole, not the year, a week's reading a
|
|
3052
|
+
// week's releases' commits.
|
|
2807
3053
|
type Attribution = AgentMerges & { releases?: string[] }
|
|
2808
3054
|
const keptAttribution = async () => ((await store.read(mergesKey))?.value as Attribution | undefined) ?? null
|
|
2809
3055
|
const attribution = async (whole: number): Promise<Attribution> => {
|
|
2810
3056
|
const [pulls, releases] = await Promise.all([keptPulls(deps, repoId, fullName, token, whole),
|
|
2811
|
-
releaseCommits(fullName, token,
|
|
3057
|
+
releaseCommits(fullName, token, whole, historyChanges(deps, repoId))])
|
|
2812
3058
|
return { ...agentMerges(pulls, first), releases }
|
|
2813
3059
|
}
|
|
2814
3060
|
// Since when every pull request merged is read (repository-pulls.ts
|
|
@@ -2816,12 +3062,13 @@ async function pulseCommits(deps: ManagedHttpDeps, repoId: number, fullName: str
|
|
|
2816
3062
|
const readTo = (read: Attribution) => pullsReadTo(read)
|
|
2817
3063
|
// The year counted from its `commits` as `read` tells whose they are and
|
|
2818
3064
|
// which were pushed directly, the branch followed through those it was
|
|
2819
|
-
// read on past (`passed`, yearCommits) too, whole from when
|
|
2820
|
-
//
|
|
2821
|
-
//
|
|
2822
|
-
|
|
3065
|
+
// read on past (`passed`, yearCommits) too, whole from when the commits,
|
|
3066
|
+
// the pull requests merged and, where it read them since later than the
|
|
3067
|
+
// year, the releases (`since`) are all read (`from`, none where all are);
|
|
3068
|
+
// kept by its head and as the year last counted.
|
|
3069
|
+
const count = async ({ commits, passed, from }: { commits: HistoryCommit[]; passed: HistoryCommit[]; from: number | null }, read: Attribution, since: number) => {
|
|
2823
3070
|
const walked = [...commits, ...passed]
|
|
2824
|
-
const whole = Math.max(from ?? Number.NEGATIVE_INFINITY, readTo(read))
|
|
3071
|
+
const whole = Math.max(from ?? Number.NEGATIVE_INFINITY, readTo(read), since > first ? since : Number.NEGATIVE_INFINITY)
|
|
2825
3072
|
const stats = { ...commitStats(commits, first, agentCommits(read, walked), new Set(read.releases), directCommits(read, walked, sha)),
|
|
2826
3073
|
...Number.isFinite(whole) ? { from: whole } : {} }
|
|
2827
3074
|
await Promise.all([store.write(key, stats, Date.now()), store.write(lastKey, { first, stats }, Date.now())])
|
|
@@ -2842,28 +3089,38 @@ async function pulseCommits(deps: ManagedHttpDeps, repoId: number, fullName: str
|
|
|
2842
3089
|
const same = kept && sameMerges(read, kept) && read.releases!.join() === (kept.releases ?? []).join() && readTo(kept) <= readTo(read)
|
|
2843
3090
|
if (readTo(read) >= whole || same) return read
|
|
2844
3091
|
const year = await yearCommits(fullName, token, sha, first, historyPages(deps, repoId, true), globalThis.fetch, whole)
|
|
2845
|
-
if (year.complete) recounted.stats = await count(year, read)
|
|
3092
|
+
if (year.complete) recounted.stats = await count(year, read, whole)
|
|
2846
3093
|
return read
|
|
2847
|
-
}, () => true).catch((err: unknown) => { console.warn('managed: pulse attribution read failed:', err) })
|
|
3094
|
+
}, () => true, false, stampOf(deps, repoId, fullName, token, today())).catch((err: unknown) => { console.warn('managed: pulse attribution read failed:', err) })
|
|
2848
3095
|
return recounted.stats ?? stats
|
|
2849
3096
|
}
|
|
2850
3097
|
// A year kept, read whole as far back as the period needs, as it was;
|
|
2851
3098
|
// else its commits and the pull requests merged read on together as far
|
|
2852
|
-
// as it needs, and counted once both are
|
|
2853
|
-
|
|
2854
|
-
|
|
3099
|
+
// as it needs, and counted once both are, how many of its commits are read
|
|
3100
|
+
// told as they are (`progress`); but where they are too many to read
|
|
3101
|
+
// since no later than it needs (kept apart, `within`, one kept before
|
|
3102
|
+
// with the year's own the year's), GitHub's statistics, a week's commits
|
|
3103
|
+
// read where the year's are too many.
|
|
3104
|
+
const tooManyKey = `${key}\0too-many`
|
|
3105
|
+
const [held, tooMany] = await Promise.all([store.read(key), store.read(tooManyKey)]).then(([year, marked]) => [
|
|
3106
|
+
year?.value as PulseStats | { tooMany: true } | undefined, marked?.value as { within?: unknown } | undefined] as const)
|
|
3107
|
+
if (held && !('tooMany' in held) && (held.from === undefined || held.from <= need)) {
|
|
2855
3108
|
return { stats: await relabelled(held), reading: null, pulls: null, pending: false, kept: head.kept }
|
|
2856
3109
|
}
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
3110
|
+
const tooManyWithin = typeof tooMany?.within === 'number' ? tooMany.within : held && 'tooMany' in held ? first : Number.NEGATIVE_INFINITY
|
|
3111
|
+
// One kept with the year's own before, kept apart now, before a shorter
|
|
3112
|
+
// period's count takes its place.
|
|
3113
|
+
if (!tooMany && held && 'tooMany' in held) await store.write(tooManyKey, { within: first }, Date.now())
|
|
3114
|
+
if (need > tooManyWithin) {
|
|
3115
|
+
const [year, read] = await Promise.all([yearCommits(fullName, token, sha, first, historyPages(deps, repoId, true), globalThis.fetch, need,
|
|
3116
|
+
commits => { if (progress) progress.commits = commits }), attribution(need)])
|
|
2860
3117
|
await store.write(mergesKey, read, Date.now())
|
|
2861
3118
|
if (year.complete) {
|
|
2862
3119
|
if (readTo(read) >= need) return { stats: await last(), reading: null, pulls: read.next?.read ?? 0, pending: false, kept: false }
|
|
2863
|
-
return { stats: await count(year, read), reading: null, pulls: null, pending: false, kept: false }
|
|
3120
|
+
return { stats: await count(year, read, need), reading: null, pulls: null, pending: false, kept: false }
|
|
2864
3121
|
}
|
|
2865
3122
|
if (!year.tooMany) return { stats: await last(), reading: year.read, pulls: null, pending: false, kept: false }
|
|
2866
|
-
await store.write(
|
|
3123
|
+
await store.write(tooManyKey, { within: need }, Date.now())
|
|
2867
3124
|
}
|
|
2868
3125
|
// Its commits and lines read together, kept as one: never the one GitHub
|
|
2869
3126
|
// has counted since a push beside the other kept from before it.
|
|
@@ -4343,8 +4600,11 @@ async function npmReviewPublicationLimit(deps: ManagedHttpDeps, reader: NpmReade
|
|
|
4343
4600
|
// Reviews are scoped to the bytes actually displayed, including the base:
|
|
4344
4601
|
// each change but a revocation loads both versions' files, of tarballs held
|
|
4345
4602
|
// to their integrity and to their shasum where npm tells one (npm-loads.ts),
|
|
4346
|
-
// and the review keeps those shasums.
|
|
4347
|
-
//
|
|
4603
|
+
// and the review keeps those shasums. A patched version's (`patchHash`) are
|
|
4604
|
+
// its files as its patch changes them, the patch read where the request says
|
|
4605
|
+
// (`patchRepoId`, `patchCommit`, `patchPath`), of the whole of a repository
|
|
4606
|
+
// its reader reads (npmPatchOf), which the review keeps. Access to both
|
|
4607
|
+
// versions is rechecked before returning or saving annotations.
|
|
4348
4608
|
async function handleNpmReviews(req: IncomingMessage, res: ServerResponse, deps: ManagedHttpDeps, cookie: string | undefined, params: URLSearchParams): Promise<void> {
|
|
4349
4609
|
const method = req.method ?? 'GET'
|
|
4350
4610
|
if (!['GET', 'POST'].includes(method)) { send405(res, 'GET, POST'); return }
|
|
@@ -4367,13 +4627,17 @@ async function handleNpmReviews(req: IncomingMessage, res: ServerResponse, deps:
|
|
|
4367
4627
|
const basePackage = field('basePackage') ?? null
|
|
4368
4628
|
const baseVersion = field('baseVersion') ?? null
|
|
4369
4629
|
const baseIntegrity = field('baseIntegrity') ?? null
|
|
4630
|
+
const patchHash = field('patchHash') ?? null
|
|
4631
|
+
// Where its patch is, where it names that: its hash alone where only listed.
|
|
4632
|
+
const patch = patchFields(field)
|
|
4370
4633
|
const validIntegrity = (value: unknown): value is string => typeof value === 'string' && value.length > 0 && value.length <= 1024
|
|
4371
4634
|
if (field('ecosystem') !== 'npm' || !isNpmPackageName(name) || !isNpmPackageSpec(version) || !validIntegrity(integrity)
|
|
4372
4635
|
|| (basePackage !== null || baseVersion !== null || baseIntegrity !== null)
|
|
4373
|
-
&& (!isNpmPackageName(basePackage) || !isNpmPackageSpec(baseVersion) || !validIntegrity(baseIntegrity))
|
|
4636
|
+
&& (!isNpmPackageName(basePackage) || !isNpmPackageSpec(baseVersion) || !validIntegrity(baseIntegrity))
|
|
4637
|
+
|| (patchHash !== null && !isPatchHash(patchHash)) || patch === 'invalid') {
|
|
4374
4638
|
sendJson(res, 400, { error: 'bad-review-context' }); return
|
|
4375
4639
|
}
|
|
4376
|
-
const context: ReviewContext = { ecosystem: 'npm', package: name, version, integrity,
|
|
4640
|
+
const context: ReviewContext = { ecosystem: 'npm', package: name, version, integrity, patchHash: patchHash as string | null,
|
|
4377
4641
|
basePackage: basePackage as string | null, baseVersion: baseVersion as string | null, baseIntegrity: baseIntegrity as string | null }
|
|
4378
4642
|
let change: ReviewChange | null = null
|
|
4379
4643
|
if (mutation) {
|
|
@@ -4409,6 +4673,8 @@ async function handleNpmReviews(req: IncomingMessage, res: ServerResponse, deps:
|
|
|
4409
4673
|
change = { action: 'save', reviewId: reviewId as string | null, revision: revision as number | null, entryId: entryId as string | null,
|
|
4410
4674
|
entry: { file, comment, status: status as ReviewInput['status'] } }
|
|
4411
4675
|
}
|
|
4676
|
+
// Each change but a revocation reads a patched version's patch.
|
|
4677
|
+
if (patchHash !== null && action !== 'revoke' && !patch) { sendJson(res, 400, { error: 'bad-review-context' }); return }
|
|
4412
4678
|
}
|
|
4413
4679
|
const controller = new AbortController()
|
|
4414
4680
|
const onClose = () => { if (!res.writableEnded) controller.abort() }
|
|
@@ -4424,15 +4690,22 @@ async function handleNpmReviews(req: IncomingMessage, res: ServerResponse, deps:
|
|
|
4424
4690
|
|| (base && (base.version !== baseVersion || base.dist.integrity !== baseIntegrity))) {
|
|
4425
4691
|
sendJson(res, 409, { error: 'review-content-changed' }); return
|
|
4426
4692
|
}
|
|
4427
|
-
// Each change but a revocation loads both versions' files
|
|
4428
|
-
// publication is held to them.
|
|
4429
|
-
const
|
|
4693
|
+
// Each change but a revocation loads both versions' files, and a patched
|
|
4694
|
+
// one's patch; a save or a publication is held to them.
|
|
4695
|
+
const reading = change && change.action !== 'revoke'
|
|
4696
|
+
const patchRead = reading && patch ? await npmPatchOf(deps, cookie, patch) : null
|
|
4697
|
+
if (patchRead && 'error' in patchRead) { sendRefusal(res, patchRead); return }
|
|
4698
|
+
const loads = reading ? [doc, ...base ? [base] : []].map(loadNpmFiles) : []
|
|
4430
4699
|
let tarballs: ReviewedTarballs | undefined
|
|
4431
4700
|
try {
|
|
4432
4701
|
const files = await Promise.all(loads.map(load => load.result))
|
|
4433
|
-
if (loads.length > 0)
|
|
4702
|
+
if (loads.length > 0) {
|
|
4703
|
+
tarballs = { shasum: doc.dist.shasum, baseShasum: base?.dist.shasum ?? null,
|
|
4704
|
+
...patchRead && patch ? { patch: { repoId: patch.repoId, commit: patch.commit, path: patch.path } } : {} }
|
|
4705
|
+
}
|
|
4434
4706
|
if (change?.action === 'publish' || change?.action === 'save') {
|
|
4435
|
-
const maps = files.map(list => new Map<string, unknown>(npmFileRows(
|
|
4707
|
+
const maps = files.map((list, at) => new Map<string, unknown>(npmFileRows(at === 0 && patchRead ? patchedFiles(list, patchRead.files) : list)
|
|
4708
|
+
.map(([path, size, text, digest]) => [path, text ?? { format: 'digest', size, digest }])))
|
|
4436
4709
|
const after = maps[0]!, before = maps[1] ?? new Map<string, unknown>()
|
|
4437
4710
|
const diff = computeBundleDiff(before, after, () => '.')
|
|
4438
4711
|
const changed: { path: string; basePath?: string }[] = [...diff.files.onlyBase, ...diff.files.onlyOther, ...diff.files.changed]
|
|
@@ -4551,6 +4824,12 @@ async function handleNpm(req: IncomingMessage, res: ServerResponse, deps: Manage
|
|
|
4551
4824
|
}
|
|
4552
4825
|
const diffed = path === NPM_RENAME_DIFF_PATH && params.has('path') ? renameDiffRequest(params) : null
|
|
4553
4826
|
if (path === NPM_RENAME_DIFF_PATH && params.has('path') && !diffed) { sendJson(res, 400, { error: 'bad-file' }); return }
|
|
4827
|
+
// The patch a patched version's files and comparison are of, where named
|
|
4828
|
+
// (`patchHash`, `patchRepoId`, `patchCommit`, `patchPath`), as Review reads them.
|
|
4829
|
+
const patch = patchFields(key => params.get(key))
|
|
4830
|
+
if (patch === 'invalid' || (params.has('patchHash') && !patch) || (patch && path !== NPM_PACKAGE_PATH && path !== NPM_COMPARISON_PATH)) {
|
|
4831
|
+
sendJson(res, 400, { error: 'bad-patch' }); return
|
|
4832
|
+
}
|
|
4554
4833
|
const reader = await npmReader(deps, cookie, otherName === name ? [name] : [name, otherName], { handed: true })
|
|
4555
4834
|
if (reader == null) { sendJson(res, 401, { error: 'unauthenticated' }); return }
|
|
4556
4835
|
const controller = new AbortController()
|
|
@@ -4678,15 +4957,20 @@ async function handleNpm(req: IncomingMessage, res: ServerResponse, deps: Manage
|
|
|
4678
4957
|
// same package), as Compare and Review pair them, with what their
|
|
4679
4958
|
// Overview reads beside them (comparison.ts): the version shown against
|
|
4680
4959
|
// the one compared with, or `(empty)`, none, as Review reads a first
|
|
4681
|
-
// version
|
|
4960
|
+
// version; as `patch` changes its files, where named, the patch read
|
|
4961
|
+
// first (npmPatchOf). Read of both, each package's access checked again
|
|
4962
|
+
// once worked out.
|
|
4682
4963
|
if (path === NPM_COMPARISON_PATH) {
|
|
4683
4964
|
const old = otherSpec === '(empty)' ? null : await otherDoc()
|
|
4965
|
+
const patched = patch ? await npmPatchOf(deps, cookie, patch) : null
|
|
4966
|
+
if (patched && 'error' in patched) { sendRefusal(res, patched); return }
|
|
4684
4967
|
res.setTimeout(NPM_RESPONSE_IDLE_MS, () => res.destroy())
|
|
4685
4968
|
const loads = [doc, ...old ? [old] : []].map(loadNpmFiles)
|
|
4686
4969
|
let body
|
|
4687
4970
|
try {
|
|
4688
4971
|
const [files, oldFiles] = await Promise.all(loads.map(load => load.result))
|
|
4689
|
-
body = await comparisonBody(old ? npmComparedSide(old.dist.integrity, oldFiles!) : emptySide(),
|
|
4972
|
+
body = await comparisonBody(old ? npmComparedSide(old.dist.integrity, oldFiles!) : emptySide(),
|
|
4973
|
+
npmComparedSide(doc.dist.integrity, patched ? patchedFiles(files!, patched.files) : files!))
|
|
4690
4974
|
} finally { for (const load of loads) load.release() }
|
|
4691
4975
|
const current = await npmReader(deps, cookie, otherName === name ? [name] : [name, otherName])
|
|
4692
4976
|
if (current == null) { body.stream.destroy(); sendJson(res, 401, { error: 'unauthenticated' }); return }
|
|
@@ -4744,6 +5028,20 @@ async function handleNpm(req: IncomingMessage, res: ServerResponse, deps: Manage
|
|
|
4744
5028
|
await sendEncoded(req, res, diff, 'application/json')
|
|
4745
5029
|
return
|
|
4746
5030
|
}
|
|
5031
|
+
// A patched version's files, as its patch changes them (npm-patch.ts),
|
|
5032
|
+
// its patch read first (npmPatchOf), as Review reads them beside the
|
|
5033
|
+
// version's own; none of what is asked beside those.
|
|
5034
|
+
if (patch) {
|
|
5035
|
+
const patched = await npmPatchOf(deps, cookie, patch)
|
|
5036
|
+
if ('error' in patched) { sendRefusal(res, patched); return }
|
|
5037
|
+
const load = loadNpmFiles(doc)
|
|
5038
|
+
let body
|
|
5039
|
+
try { body = await patchedVersionBody(doc, patch, patchedFiles(await load.result, patched.files)) } finally { load.release() }
|
|
5040
|
+
if (!(await recheck(doc.private))) { body.stream.destroy(); return }
|
|
5041
|
+
res.setTimeout(NPM_RESPONSE_IDLE_MS, () => res.destroy())
|
|
5042
|
+
await sendEncoded(req, res, body, 'application/json')
|
|
5043
|
+
return
|
|
5044
|
+
}
|
|
4747
5045
|
// The load stays held until this response is written or abandoned, and
|
|
4748
5046
|
// a reader who stops reading it gives it up after a while. One already
|
|
4749
5047
|
// gone takes none, as its close has passed.
|
|
@@ -4828,6 +5126,10 @@ export function createManagedRequestHandler(deps: ManagedHttpDeps): Handler {
|
|
|
4828
5126
|
// Kept reads' prunes run behind their writes: tracked, so the database
|
|
4829
5127
|
// outlives them, and on serverless kept alive past the answer.
|
|
4830
5128
|
ownKeptPrunes(db, prune => { track(prune); deps.waitUntil?.(prune) })
|
|
5129
|
+
// GitHub's tagged answers kept beyond this process, for an instance
|
|
5130
|
+
// starting anew to ask again If-None-Match (github-requests.ts).
|
|
5131
|
+
const tags = githubTagStore<TaggedRead>(db, config.debug)
|
|
5132
|
+
keepTaggedReadsIn({ read: async key => (await tags.read(key))?.value ?? null, write: (key, read) => tags.write(key, read, Date.now()) })
|
|
4831
5133
|
|
|
4832
5134
|
async function route(req: IncomingMessage, res: ServerResponse): Promise<void> {
|
|
4833
5135
|
const url = new URL(req.url ?? '/', 'http://localhost')
|
|
@@ -5240,10 +5542,12 @@ export function createManagedRequestHandler(deps: ManagedHttpDeps): Handler {
|
|
|
5240
5542
|
if (method !== 'GET') { send405(res, 'GET'); return }
|
|
5241
5543
|
const s = workspaceSession!
|
|
5242
5544
|
const integrity = url.searchParams.get('integrity'), name = url.searchParams.get('package'), version = url.searchParams.get('version')
|
|
5243
|
-
|
|
5545
|
+
const patchHash = url.searchParams.get('patchHash')
|
|
5546
|
+
if (!isNpmPackageName(name) || !isNpmPackageSpec(version) || !integrity || integrity.length > 1024
|
|
5547
|
+
|| (patchHash !== null && !isPatchHash(patchHash))) {
|
|
5244
5548
|
sendJson(res, 400, { error: 'bad-review-context' }); return
|
|
5245
5549
|
}
|
|
5246
|
-
const review = await deps.db.getLatestOwnDependencyReview(s.user.id, { ecosystem: 'npm', package: name, version, integrity })
|
|
5550
|
+
const review = await deps.db.getLatestOwnDependencyReview(s.user.id, { ecosystem: 'npm', package: name, version, integrity, patchHash })
|
|
5247
5551
|
const visible = await readableNpmReviews(res, deps, cookie, s.user.id, review ? [review] : [], s.user)
|
|
5248
5552
|
if (visible) sendJson(res, 200, { review: (await withFormatted(deps.reviewedDependencies ?? null, s.user.id, visible))[0] ?? null })
|
|
5249
5553
|
return
|