@verdaccio/store 7.0.0-next.6 → 8.0.0-next-8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +340 -0
- package/README.md +24 -57
- package/build/index.d.ts +0 -1
- package/build/index.js +0 -12
- package/build/index.js.map +1 -1
- package/build/lib/star-utils.js.map +1 -1
- package/build/lib/storage-utils.d.ts +3 -0
- package/build/lib/storage-utils.js +35 -2
- package/build/lib/storage-utils.js.map +1 -1
- package/build/lib/versions-utils.d.ts +14 -0
- package/build/lib/versions-utils.js +52 -1
- package/build/lib/versions-utils.js.map +1 -1
- package/build/local-storage.js +3 -3
- package/build/local-storage.js.map +1 -1
- package/build/storage.d.ts +14 -15
- package/build/storage.js +198 -161
- package/build/storage.js.map +1 -1
- package/build/type.d.ts +5 -0
- package/build/type.js.map +1 -1
- package/jest.config.js +2 -2
- package/package.json +17 -16
- package/src/index.ts +0 -1
- package/src/lib/storage-utils.ts +40 -2
- package/src/lib/versions-utils.ts +50 -1
- package/src/local-storage.ts +3 -2
- package/src/storage.ts +228 -149
- package/src/type.ts +6 -0
- package/test/fixtures/config/publishWithOwnerAndCheck.yaml +18 -0
- package/test/fixtures/config/publishWithOwnerDefault.yaml +15 -0
- package/test/fixtures/config/syncDoubleUplinksMetadata.yaml +2 -18
- package/test/search.spec.ts +28 -7
- package/test/storage-utils.spec.ts +1 -0
- package/test/storage.spec.ts +466 -80
- package/test/versions.spec.ts +58 -1
- package/build/lib/TransFormResults.d.ts +0 -15
- package/build/lib/TransFormResults.js +0 -51
- package/build/lib/TransFormResults.js.map +0 -1
- package/build/lib/search-utils.d.ts +0 -4
- package/build/lib/search-utils.js +0 -50
- package/build/lib/search-utils.js.map +0 -1
- package/build/lib/uplink-util.d.ts +0 -10
- package/build/lib/uplink-util.js +0 -43
- package/build/lib/uplink-util.js.map +0 -1
- package/src/lib/TransFormResults.ts +0 -42
- package/src/lib/search-utils.ts +0 -50
- package/src/lib/uplink-util.ts +0 -42
- /package/test/fixtures/config/{getTarballNext-getupstream.yaml → getTarball-getupstream.yaml} +0 -0
package/src/storage.ts
CHANGED
|
@@ -2,7 +2,7 @@ import assert from 'assert';
|
|
|
2
2
|
import buildDebug from 'debug';
|
|
3
3
|
import _, { isEmpty, isNil } from 'lodash';
|
|
4
4
|
import { basename } from 'path';
|
|
5
|
-
import { PassThrough, Readable, Transform
|
|
5
|
+
import { PassThrough, Readable, Transform } from 'stream';
|
|
6
6
|
import { pipeline } from 'stream/promises';
|
|
7
7
|
import { default as URL } from 'url';
|
|
8
8
|
|
|
@@ -10,9 +10,12 @@ import { hasProxyTo } from '@verdaccio/config';
|
|
|
10
10
|
import {
|
|
11
11
|
API_ERROR,
|
|
12
12
|
API_MESSAGE,
|
|
13
|
+
DEFAULT_USER,
|
|
13
14
|
DIST_TAGS,
|
|
14
15
|
HEADER_TYPE,
|
|
15
16
|
HTTP_STATUS,
|
|
17
|
+
MAINTAINERS,
|
|
18
|
+
PLUGIN_CATEGORY,
|
|
16
19
|
SUPPORT_ERRORS,
|
|
17
20
|
USERS,
|
|
18
21
|
errorUtils,
|
|
@@ -22,12 +25,22 @@ import {
|
|
|
22
25
|
validatioUtils,
|
|
23
26
|
} from '@verdaccio/core';
|
|
24
27
|
import { asyncLoadPlugin } from '@verdaccio/loaders';
|
|
25
|
-
import { logger } from '@verdaccio/logger';
|
|
26
|
-
import { IProxy, ISyncUplinksOptions, ProxySearchParams, ProxyStorage } from '@verdaccio/proxy';
|
|
27
28
|
import {
|
|
29
|
+
IProxy,
|
|
30
|
+
ISyncUplinksOptions,
|
|
31
|
+
ProxyInstanceList,
|
|
32
|
+
ProxySearchParams,
|
|
33
|
+
ProxyStorage,
|
|
34
|
+
setupUpLinks,
|
|
35
|
+
updateVersionsHiddenUpLinkNext,
|
|
36
|
+
} from '@verdaccio/proxy';
|
|
37
|
+
import Search from '@verdaccio/search';
|
|
38
|
+
import {
|
|
39
|
+
TarballDetails,
|
|
28
40
|
convertDistRemoteToLocalTarballUrls,
|
|
29
41
|
convertDistVersionToLocalTarballsUrl,
|
|
30
42
|
extractTarballFromUrl,
|
|
43
|
+
getTarballDetails,
|
|
31
44
|
} from '@verdaccio/tarball';
|
|
32
45
|
import {
|
|
33
46
|
AbbreviatedManifest,
|
|
@@ -52,12 +65,9 @@ import {
|
|
|
52
65
|
UpdateManifestOptions,
|
|
53
66
|
cleanUpReadme,
|
|
54
67
|
isDeprecatedManifest,
|
|
55
|
-
mapManifestToSearchPackageBody,
|
|
56
68
|
tagVersion,
|
|
57
69
|
tagVersionNext,
|
|
58
70
|
} from '.';
|
|
59
|
-
import { TransFormResults } from './lib/TransFormResults';
|
|
60
|
-
import { removeDuplicates } from './lib/search-utils';
|
|
61
71
|
import { isPublishablePackage } from './lib/star-utils';
|
|
62
72
|
import { isExecutingStarCommand } from './lib/star-utils';
|
|
63
73
|
import {
|
|
@@ -66,16 +76,16 @@ import {
|
|
|
66
76
|
generatePackageTemplate,
|
|
67
77
|
generateRevision,
|
|
68
78
|
getLatestReadme,
|
|
79
|
+
mapManifestToSearchPackageBody,
|
|
69
80
|
mergeUplinkTimeIntoLocalNext,
|
|
70
81
|
mergeVersions,
|
|
71
82
|
normalizeDistTags,
|
|
72
83
|
normalizePackage,
|
|
73
84
|
updateUpLinkMetadata,
|
|
74
85
|
} from './lib/storage-utils';
|
|
75
|
-
import {
|
|
76
|
-
import { getVersion } from './lib/versions-utils';
|
|
86
|
+
import { getVersion, removeLowerVersions } from './lib/versions-utils';
|
|
77
87
|
import { LocalStorage } from './local-storage';
|
|
78
|
-
import { IGetPackageOptionsNext, StarManifestBody } from './type';
|
|
88
|
+
import { IGetPackageOptionsNext, OwnerManifestBody, StarManifestBody } from './type';
|
|
79
89
|
|
|
80
90
|
const debug = buildDebug('verdaccio:storage');
|
|
81
91
|
|
|
@@ -90,10 +100,12 @@ class Storage {
|
|
|
90
100
|
public readonly config: Config;
|
|
91
101
|
public readonly logger: Logger;
|
|
92
102
|
public readonly uplinks: ProxyInstanceList;
|
|
93
|
-
|
|
103
|
+
private searchService: Search;
|
|
104
|
+
public constructor(config: Config, logger: Logger) {
|
|
94
105
|
this.config = config;
|
|
95
|
-
this.uplinks = setupUpLinks(config);
|
|
96
106
|
this.logger = logger.child({ module: 'storage' });
|
|
107
|
+
this.uplinks = setupUpLinks(config, this.logger);
|
|
108
|
+
this.searchService = new Search(config, this.logger);
|
|
97
109
|
this.filters = null;
|
|
98
110
|
// @ts-ignore
|
|
99
111
|
this.localStorage = null;
|
|
@@ -109,7 +121,7 @@ class Storage {
|
|
|
109
121
|
*/
|
|
110
122
|
public async changePackage(name: string, metadata: Manifest, revision: string): Promise<void> {
|
|
111
123
|
debug('change existing package for package %o revision %o', name, revision);
|
|
112
|
-
debug(`change manifest tags for %o revision %
|
|
124
|
+
debug(`change manifest tags for %o revision %o`, name, revision);
|
|
113
125
|
if (
|
|
114
126
|
!validatioUtils.isObject(metadata.versions) ||
|
|
115
127
|
!validatioUtils.isObject(metadata[DIST_TAGS])
|
|
@@ -118,7 +130,7 @@ class Storage {
|
|
|
118
130
|
throw errorUtils.getBadData();
|
|
119
131
|
}
|
|
120
132
|
|
|
121
|
-
debug(`change manifest
|
|
133
|
+
debug(`change manifest updating manifest for %o`, name);
|
|
122
134
|
await this.updatePackage(name, async (localData: Manifest): Promise<Manifest> => {
|
|
123
135
|
// eslint-disable-next-line guard-for-in
|
|
124
136
|
for (const version in localData.versions) {
|
|
@@ -155,13 +167,14 @@ class Storage {
|
|
|
155
167
|
|
|
156
168
|
localData[USERS] = metadata[USERS];
|
|
157
169
|
localData[DIST_TAGS] = metadata[DIST_TAGS];
|
|
170
|
+
localData[MAINTAINERS] = metadata[MAINTAINERS];
|
|
158
171
|
return localData;
|
|
159
172
|
});
|
|
160
173
|
}
|
|
161
174
|
|
|
162
|
-
public async removePackage(name: string, revision): Promise<void> {
|
|
175
|
+
public async removePackage(name: string, revision: string, username: string): Promise<void> {
|
|
163
176
|
debug('remove package %o', name);
|
|
164
|
-
await this.removePackageByRevision(name, revision);
|
|
177
|
+
await this.removePackageByRevision(name, revision, username);
|
|
165
178
|
}
|
|
166
179
|
|
|
167
180
|
/**
|
|
@@ -171,8 +184,13 @@ class Storage {
|
|
|
171
184
|
versions, i.e. package version should be unpublished first.
|
|
172
185
|
Used storage: local (write)
|
|
173
186
|
*/
|
|
174
|
-
|
|
175
|
-
|
|
187
|
+
public async removeTarball(
|
|
188
|
+
name: string,
|
|
189
|
+
filename: string,
|
|
190
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
191
|
+
_revision: string,
|
|
192
|
+
username: string
|
|
193
|
+
): Promise<Manifest> {
|
|
176
194
|
debug('remove tarball %s for %s', filename, name);
|
|
177
195
|
assert(validatioUtils.validateName(filename));
|
|
178
196
|
const storage: pluginUtils.StorageHandler = this.getPrivatePackageStorage(name);
|
|
@@ -187,6 +205,9 @@ class Storage {
|
|
|
187
205
|
if (!cacheManifest._attachments[filename]) {
|
|
188
206
|
throw errorUtils.getNotFound('no such file available');
|
|
189
207
|
}
|
|
208
|
+
|
|
209
|
+
// check if logged in user is allowed to remove tarball
|
|
210
|
+
await this.checkAllowedToChangePackage(cacheManifest, username);
|
|
190
211
|
} catch (err: any) {
|
|
191
212
|
if (err.code === noSuchFile) {
|
|
192
213
|
throw errorUtils.getNotFound();
|
|
@@ -220,63 +241,19 @@ class Storage {
|
|
|
220
241
|
/**
|
|
221
242
|
* Handle search on packages and proxies.
|
|
222
243
|
* Iterate all proxies configured and search in all endpoints in v2 and pipe all responses
|
|
223
|
-
*
|
|
244
|
+
* once the proxies request has finished search in local storage for all packages
|
|
224
245
|
* (privated and cached).
|
|
225
246
|
*/
|
|
226
247
|
public async search(options: ProxySearchParams): Promise<searchUtils.SearchPackageItem[]> {
|
|
227
|
-
|
|
228
|
-
const
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
}
|
|
237
|
-
return this.consumeSearchStream(uplinkId, uplink, options, streamPassThrough);
|
|
238
|
-
});
|
|
239
|
-
|
|
240
|
-
try {
|
|
241
|
-
debug('searching on %s uplinks...', searchUplinksStreams?.length);
|
|
242
|
-
// only process those streams end successfully, if all request fails
|
|
243
|
-
// just include local storage results (if local fails then return 500)
|
|
244
|
-
await Promise.allSettled([...searchUplinksStreams]);
|
|
245
|
-
debug('searching all uplinks done');
|
|
246
|
-
} catch (err: any) {
|
|
247
|
-
this.logger.error({ err: err?.message }, ' error on uplinks search @{err}');
|
|
248
|
-
streamPassThrough.emit('error', err);
|
|
249
|
-
}
|
|
250
|
-
debug('search local');
|
|
251
|
-
try {
|
|
252
|
-
await this.searchCachedPackages(streamPassThrough, options.query as searchUtils.SearchQuery);
|
|
253
|
-
} catch (err: any) {
|
|
254
|
-
this.logger.error({ err: err?.message }, ' error on local search @{err}');
|
|
255
|
-
streamPassThrough.emit('error', err);
|
|
256
|
-
}
|
|
257
|
-
const data: searchUtils.SearchPackageItem[] = [];
|
|
258
|
-
const outPutStream = new PassThrough({ objectMode: true });
|
|
259
|
-
streamPipeline(streamPassThrough, transformResults, outPutStream, (err: any) => {
|
|
260
|
-
if (err) {
|
|
261
|
-
this.logger.error({ err: err?.message }, ' error on search @{err}');
|
|
262
|
-
throw errorUtils.getInternalError(err ? err.message : 'unknown search error');
|
|
263
|
-
} else {
|
|
264
|
-
debug('pipeline succeeded');
|
|
265
|
-
}
|
|
266
|
-
});
|
|
267
|
-
|
|
268
|
-
outPutStream.on('data', (chunk) => {
|
|
269
|
-
data.push(chunk);
|
|
270
|
-
});
|
|
271
|
-
|
|
272
|
-
return new Promise((resolve) => {
|
|
273
|
-
outPutStream.on('finish', async () => {
|
|
274
|
-
const searchFinalResults: searchUtils.SearchPackageItem[] = removeDuplicates(data);
|
|
275
|
-
debug('search stream total results: %o', searchFinalResults.length);
|
|
276
|
-
return resolve(searchFinalResults);
|
|
277
|
-
});
|
|
278
|
-
debug('search done');
|
|
279
|
-
});
|
|
248
|
+
debug('search on cache packages');
|
|
249
|
+
const cachePackages = await this.getCachedPackages(options.query);
|
|
250
|
+
debug('search found on cache packages %o', cachePackages.length);
|
|
251
|
+
const remotePackages = await this.searchService.search(options);
|
|
252
|
+
debug('search found on remote packages %o', remotePackages.length);
|
|
253
|
+
const totalResults = [...cachePackages, ...remotePackages];
|
|
254
|
+
const uniqueResults = removeLowerVersions(totalResults);
|
|
255
|
+
debug('unique results %o', uniqueResults.length);
|
|
256
|
+
return uniqueResults;
|
|
280
257
|
}
|
|
281
258
|
|
|
282
259
|
private async getTarballFromUpstream(name: string, filename: string, { signal }) {
|
|
@@ -382,7 +359,7 @@ class Storage {
|
|
|
382
359
|
// should not be the case
|
|
383
360
|
const passThroughRemoteStream = new PassThrough();
|
|
384
361
|
// ensure get the latest data
|
|
385
|
-
const [updatedManifest] = await this.
|
|
362
|
+
const [updatedManifest] = await this.syncUplinksMetadata(name, cachedManifest, {
|
|
386
363
|
uplinksLook: true,
|
|
387
364
|
});
|
|
388
365
|
const distFile = (updatedManifest as Manifest)._distfiles[filename];
|
|
@@ -425,7 +402,7 @@ class Storage {
|
|
|
425
402
|
* @param param2
|
|
426
403
|
* @returns
|
|
427
404
|
*/
|
|
428
|
-
public async
|
|
405
|
+
public async getTarball(name: string, filename: string, { signal }): Promise<PassThrough> {
|
|
429
406
|
debug('get tarball for package %o filename %o', name, filename);
|
|
430
407
|
// TODO: check if isOpen is need it after all.
|
|
431
408
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
@@ -518,6 +495,17 @@ class Storage {
|
|
|
518
495
|
public async getPackageManifest(options: IGetPackageOptionsNext): Promise<Manifest> {
|
|
519
496
|
// convert dist remotes to local bars
|
|
520
497
|
const [manifest] = await this.getPackageNext(options);
|
|
498
|
+
|
|
499
|
+
// If change access is requested (?write=true), then check if logged in user is allowed to change package
|
|
500
|
+
if (options.byPassCache === true) {
|
|
501
|
+
try {
|
|
502
|
+
await this.checkAllowedToChangePackage(manifest, options.requestOptions.username);
|
|
503
|
+
} catch (error: any) {
|
|
504
|
+
this.logger.error({ error: error.message }, 'getting package has failed: @{error}');
|
|
505
|
+
throw errorUtils.getBadRequest(error.message);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
|
|
521
509
|
const convertedManifest = convertDistRemoteToLocalTarballUrls(
|
|
522
510
|
manifest,
|
|
523
511
|
options.requestOptions,
|
|
@@ -536,7 +524,6 @@ class Storage {
|
|
|
536
524
|
const _version_abbreviated = {
|
|
537
525
|
name: _version.name,
|
|
538
526
|
version: _version.version,
|
|
539
|
-
description: _version.description,
|
|
540
527
|
deprecated: _version.deprecated,
|
|
541
528
|
bin: _version.bin,
|
|
542
529
|
dist: _version.dist,
|
|
@@ -548,6 +535,10 @@ class Storage {
|
|
|
548
535
|
peerDependencies: _version.peerDependencies,
|
|
549
536
|
optionalDependencies: _version.optionalDependencies,
|
|
550
537
|
bundleDependencies: _version.bundleDependencies,
|
|
538
|
+
cpu: _version.cpu,
|
|
539
|
+
os: _version.os,
|
|
540
|
+
peerDependenciesMeta: _version.peerDependenciesMeta,
|
|
541
|
+
acceptDependencies: _version.acceptDependencies,
|
|
551
542
|
// npm cli specifics
|
|
552
543
|
_hasShrinkwrap: _version._hasShrinkwrap,
|
|
553
544
|
hasInstallScript: _version.hasInstallScript,
|
|
@@ -656,7 +647,7 @@ class Storage {
|
|
|
656
647
|
*/
|
|
657
648
|
public async init(config: Config): Promise<void> {
|
|
658
649
|
if (this.localStorage === null) {
|
|
659
|
-
this.localStorage = new LocalStorage(this.config, logger);
|
|
650
|
+
this.localStorage = new LocalStorage(this.config, this.logger);
|
|
660
651
|
await this.localStorage.init();
|
|
661
652
|
debug('local init storage initialized');
|
|
662
653
|
await this.localStorage.getSecret(config);
|
|
@@ -674,35 +665,14 @@ class Storage {
|
|
|
674
665
|
(plugin: pluginUtils.ManifestFilter<Config>) => {
|
|
675
666
|
return typeof plugin.filter_metadata !== 'undefined';
|
|
676
667
|
},
|
|
677
|
-
this.config?.serverSettings?.pluginPrefix
|
|
668
|
+
this.config?.serverSettings?.pluginPrefix,
|
|
669
|
+
PLUGIN_CATEGORY.FILTER
|
|
678
670
|
);
|
|
679
|
-
debug('filters available %o', this.filters);
|
|
671
|
+
debug('filters available %o', this.filters.length);
|
|
680
672
|
}
|
|
681
673
|
return;
|
|
682
674
|
}
|
|
683
675
|
|
|
684
|
-
/**
|
|
685
|
-
* Consume the upstream and pipe it to a transformable stream.
|
|
686
|
-
*/
|
|
687
|
-
private consumeSearchStream(
|
|
688
|
-
uplinkId: string,
|
|
689
|
-
uplink: IProxy,
|
|
690
|
-
options: ProxySearchParams,
|
|
691
|
-
searchPassThrough: PassThrough
|
|
692
|
-
): Promise<any> {
|
|
693
|
-
return uplink.search({ ...options }).then((bodyStream) => {
|
|
694
|
-
bodyStream.pipe(searchPassThrough, { end: false });
|
|
695
|
-
bodyStream.on('error', (err: any): void => {
|
|
696
|
-
logger.error(
|
|
697
|
-
{ uplinkId, err: err },
|
|
698
|
-
'search error for uplink @{uplinkId}: @{err?.message}'
|
|
699
|
-
);
|
|
700
|
-
searchPassThrough.end();
|
|
701
|
-
});
|
|
702
|
-
return new Promise((resolve) => bodyStream.on('end', resolve));
|
|
703
|
-
});
|
|
704
|
-
}
|
|
705
|
-
|
|
706
676
|
/**
|
|
707
677
|
* Retrieve a wrapper that provide access to the package location.
|
|
708
678
|
* @param {Object} pkgName package name.
|
|
@@ -734,11 +704,16 @@ class Storage {
|
|
|
734
704
|
return await storage.readTarball(filename, { signal });
|
|
735
705
|
}
|
|
736
706
|
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
707
|
+
public async getCachedPackages(
|
|
708
|
+
query?: searchUtils.SearchQuery
|
|
709
|
+
): Promise<searchUtils.SearchPackageItem[]> {
|
|
710
|
+
debug('search on each package', query);
|
|
711
|
+
const results: searchUtils.SearchPackageItem[] = [];
|
|
712
|
+
if (typeof query === 'undefined' || typeof query?.text === 'undefined') {
|
|
713
|
+
debug('search query for cached not found');
|
|
714
|
+
return results;
|
|
715
|
+
}
|
|
716
|
+
|
|
742
717
|
this.logger.info(
|
|
743
718
|
{ t: query.text, q: query.quality, p: query.popularity, m: query.maintenance, s: query.size },
|
|
744
719
|
'search by text @{t}| maintenance @{m}| quality @{q}| popularity @{p}'
|
|
@@ -746,15 +721,15 @@ class Storage {
|
|
|
746
721
|
|
|
747
722
|
if (typeof this.localStorage.getStoragePlugin().search === 'undefined') {
|
|
748
723
|
this.logger.info('plugin search not implemented yet');
|
|
749
|
-
searchStream.end();
|
|
750
724
|
} else {
|
|
751
|
-
debug('search on each package by plugin');
|
|
725
|
+
debug('search on each package by plugin query');
|
|
752
726
|
const items = await this.localStorage.getStoragePlugin().search(query);
|
|
753
727
|
try {
|
|
754
728
|
for (const searchItem of items) {
|
|
755
729
|
const manifest = await this.getPackageLocalMetadata(searchItem.package.name);
|
|
756
730
|
if (_.isEmpty(manifest?.versions) === false) {
|
|
757
731
|
const searchPackage = mapManifestToSearchPackageBody(manifest, searchItem);
|
|
732
|
+
debug('search local stream found %o', searchPackage.name);
|
|
758
733
|
const searchPackageItem: searchUtils.SearchPackageItem = {
|
|
759
734
|
package: searchPackage,
|
|
760
735
|
score: searchItem.score,
|
|
@@ -764,19 +739,25 @@ class Storage {
|
|
|
764
739
|
// FUTURE: find a better way to calculate the score
|
|
765
740
|
searchScore: 1,
|
|
766
741
|
};
|
|
767
|
-
|
|
742
|
+
results.push(searchPackageItem);
|
|
743
|
+
} else {
|
|
744
|
+
debug('local item without versions detected %s', searchItem.package.name);
|
|
768
745
|
}
|
|
769
746
|
}
|
|
770
747
|
debug('search local stream end');
|
|
771
|
-
searchStream.end();
|
|
772
748
|
} catch (err) {
|
|
773
749
|
this.logger.error({ err, query }, 'error on search by plugin @{err.message}');
|
|
774
|
-
|
|
750
|
+
throw err;
|
|
775
751
|
}
|
|
776
752
|
}
|
|
753
|
+
return results;
|
|
777
754
|
}
|
|
778
755
|
|
|
779
|
-
private async removePackageByRevision(
|
|
756
|
+
private async removePackageByRevision(
|
|
757
|
+
pkgName: string,
|
|
758
|
+
revision: string,
|
|
759
|
+
username: string
|
|
760
|
+
): Promise<void> {
|
|
780
761
|
const storage: pluginUtils.StorageHandler = this.getPrivatePackageStorage(pkgName);
|
|
781
762
|
debug('get package metadata for %o', pkgName);
|
|
782
763
|
if (typeof storage === 'undefined') {
|
|
@@ -788,10 +769,10 @@ class Storage {
|
|
|
788
769
|
manifest = normalizePackage(manifest);
|
|
789
770
|
} catch (err: any) {
|
|
790
771
|
if (err.code === STORAGE.NO_SUCH_FILE_ERROR || err.code === HTTP_STATUS.NOT_FOUND) {
|
|
791
|
-
logger.info({ pkgName, revision }, 'package not found');
|
|
772
|
+
this.logger.info({ pkgName, revision }, 'package not found');
|
|
792
773
|
throw errorUtils.getNotFound();
|
|
793
774
|
}
|
|
794
|
-
logger.error(
|
|
775
|
+
this.logger.error(
|
|
795
776
|
{ pkgName, revision, err: err.message },
|
|
796
777
|
'error @{err} while reading package @{pkgName}-{revision}'
|
|
797
778
|
);
|
|
@@ -800,6 +781,9 @@ class Storage {
|
|
|
800
781
|
|
|
801
782
|
// TODO: move this to another method
|
|
802
783
|
try {
|
|
784
|
+
// check if logged in user is allowed to remove package
|
|
785
|
+
await this.checkAllowedToChangePackage(manifest, username);
|
|
786
|
+
|
|
803
787
|
await this.localStorage.getStoragePlugin().remove(pkgName);
|
|
804
788
|
// remove each attachment
|
|
805
789
|
const attachments = Object.keys(manifest._attachments);
|
|
@@ -807,7 +791,7 @@ class Storage {
|
|
|
807
791
|
for (let attachment of attachments) {
|
|
808
792
|
debug('remove attachment %s', attachment);
|
|
809
793
|
await storage.deletePackage(attachment);
|
|
810
|
-
logger.info({ attachment }, 'attachment @{attachment} removed');
|
|
794
|
+
this.logger.info({ attachment }, 'attachment @{attachment} removed');
|
|
811
795
|
}
|
|
812
796
|
// remove package.json
|
|
813
797
|
debug('remove package.json');
|
|
@@ -815,7 +799,7 @@ class Storage {
|
|
|
815
799
|
// remove folder
|
|
816
800
|
debug('remove package folder');
|
|
817
801
|
await storage.removePackage();
|
|
818
|
-
logger.info({ pkgName }, 'package @{pkgName} removed');
|
|
802
|
+
this.logger.info({ pkgName }, 'package @{pkgName} removed');
|
|
819
803
|
} catch (err: any) {
|
|
820
804
|
this.logger.error({ err }, 'removed package has failed @{err.message}');
|
|
821
805
|
throw errorUtils.getBadData(err.message);
|
|
@@ -914,16 +898,17 @@ class Storage {
|
|
|
914
898
|
cache: true,
|
|
915
899
|
},
|
|
916
900
|
this.config,
|
|
917
|
-
logger
|
|
901
|
+
this.logger
|
|
918
902
|
);
|
|
919
903
|
}
|
|
920
904
|
return uplink;
|
|
921
905
|
}
|
|
922
906
|
|
|
923
907
|
public async updateManifest(
|
|
924
|
-
manifest: Manifest | StarManifestBody,
|
|
908
|
+
manifest: Manifest | StarManifestBody | OwnerManifestBody,
|
|
925
909
|
options: UpdateManifestOptions
|
|
926
910
|
): Promise<string | undefined> {
|
|
911
|
+
debug('update manifest %o for user %o', manifest._id, options.requestOptions.username);
|
|
927
912
|
if (isDeprecatedManifest(manifest as Manifest)) {
|
|
928
913
|
// if the manifest is deprecated, we need to update the package.json
|
|
929
914
|
await this.deprecate(manifest as Manifest, {
|
|
@@ -931,13 +916,22 @@ class Storage {
|
|
|
931
916
|
});
|
|
932
917
|
} else if (
|
|
933
918
|
isPublishablePackage(manifest as Manifest) === false &&
|
|
934
|
-
validatioUtils.isObject(manifest.users)
|
|
919
|
+
validatioUtils.isObject((manifest as StarManifestBody).users)
|
|
935
920
|
) {
|
|
936
921
|
// if user request to apply a star to the manifest
|
|
937
922
|
await this.star(manifest as StarManifestBody, {
|
|
938
923
|
...options,
|
|
939
924
|
});
|
|
940
925
|
return API_MESSAGE.PKG_CHANGED;
|
|
926
|
+
} else if (
|
|
927
|
+
isPublishablePackage(manifest as Manifest) === false &&
|
|
928
|
+
Array.isArray((manifest as OwnerManifestBody).maintainers)
|
|
929
|
+
) {
|
|
930
|
+
// if user request to change owners of package
|
|
931
|
+
await this.changeOwners(manifest as OwnerManifestBody, {
|
|
932
|
+
...options,
|
|
933
|
+
});
|
|
934
|
+
return API_MESSAGE.PKG_CHANGED;
|
|
941
935
|
} else if (validatioUtils.validatePublishSingleVersion(manifest)) {
|
|
942
936
|
// if continue, the version to be published does not exist
|
|
943
937
|
// we create a new package
|
|
@@ -951,14 +945,14 @@ class Storage {
|
|
|
951
945
|
try {
|
|
952
946
|
const { name } = mergedManifest;
|
|
953
947
|
await this.notify(mergedManifest, `${name}@${version}`);
|
|
954
|
-
logger.info({ name, version }, 'notify for @{name}@@{version} has been sent');
|
|
948
|
+
this.logger.info({ name, version }, 'notify for @{name}@@{version} has been sent');
|
|
955
949
|
} catch (error: any) {
|
|
956
|
-
logger.error({ error: error.message }, 'notify batch service has failed: @{error}');
|
|
950
|
+
this.logger.error({ error: error.message }, 'notify batch service has failed: @{error}');
|
|
957
951
|
}
|
|
958
952
|
return message;
|
|
959
953
|
} else {
|
|
960
954
|
debug('invalid body format');
|
|
961
|
-
logger.
|
|
955
|
+
this.logger.warn(
|
|
962
956
|
{ packageName: options.name },
|
|
963
957
|
`wrong package format on publish a package @{packageName}`
|
|
964
958
|
);
|
|
@@ -1019,6 +1013,36 @@ class Storage {
|
|
|
1019
1013
|
return API_MESSAGE.PKG_CHANGED;
|
|
1020
1014
|
}
|
|
1021
1015
|
|
|
1016
|
+
private async changeOwners(
|
|
1017
|
+
manifest: OwnerManifestBody,
|
|
1018
|
+
options: UpdateManifestOptions
|
|
1019
|
+
): Promise<string> {
|
|
1020
|
+
const { maintainers } = manifest;
|
|
1021
|
+
const { requestOptions, name } = options;
|
|
1022
|
+
debug('change owners of %o', name);
|
|
1023
|
+
const { username } = requestOptions;
|
|
1024
|
+
if (!username) {
|
|
1025
|
+
throw errorUtils.getBadRequest('update owners only allowed for logged in users');
|
|
1026
|
+
}
|
|
1027
|
+
if (!maintainers || maintainers.length === 0) {
|
|
1028
|
+
throw errorUtils.getBadRequest('maintainers field is required and must not be empty');
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
const localPackage = await this.getPackageManifest({
|
|
1032
|
+
name,
|
|
1033
|
+
requestOptions,
|
|
1034
|
+
uplinksLook: false,
|
|
1035
|
+
});
|
|
1036
|
+
|
|
1037
|
+
await this.changePackage(
|
|
1038
|
+
name,
|
|
1039
|
+
{ ...localPackage, maintainers: maintainers as Author[] },
|
|
1040
|
+
options.revision as string
|
|
1041
|
+
);
|
|
1042
|
+
|
|
1043
|
+
return API_MESSAGE.PKG_CHANGED;
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1022
1046
|
/**
|
|
1023
1047
|
* Get local package, on fails return null.
|
|
1024
1048
|
* Errors are considered package not found.
|
|
@@ -1076,7 +1100,8 @@ class Storage {
|
|
|
1076
1100
|
options: PublishOptions
|
|
1077
1101
|
): Promise<[Manifest, string, string]> {
|
|
1078
1102
|
const { name } = options;
|
|
1079
|
-
|
|
1103
|
+
const username = options.requestOptions.username;
|
|
1104
|
+
debug('publishing a new package for %o as %o', name, username);
|
|
1080
1105
|
let successResponseMessage;
|
|
1081
1106
|
const manifest: Manifest = { ...validatioUtils.normalizeMetadata(body, name) };
|
|
1082
1107
|
const { _attachments, versions } = manifest;
|
|
@@ -1098,43 +1123,47 @@ class Storage {
|
|
|
1098
1123
|
|
|
1099
1124
|
try {
|
|
1100
1125
|
// we check if package exist already locally
|
|
1101
|
-
const
|
|
1126
|
+
const localManifest = await this.getPackagelocalByNameNext(name);
|
|
1102
1127
|
// if continue, the version to be published does not exist
|
|
1103
|
-
if (
|
|
1104
|
-
debug('%s version %s already exists', name, versionToPublish);
|
|
1128
|
+
if (localManifest?.versions[versionToPublish] != null) {
|
|
1129
|
+
debug('%s version %s already exists (locally)', name, versionToPublish);
|
|
1105
1130
|
throw errorUtils.getConflict();
|
|
1106
1131
|
}
|
|
1107
1132
|
const uplinksLook = this.config?.publish?.allow_offline === false;
|
|
1108
1133
|
// if execution get here, package does not exist locally, we search upstream
|
|
1109
1134
|
const remoteManifest = await this.checkPackageRemote(name, uplinksLook);
|
|
1110
1135
|
if (remoteManifest?.versions[versionToPublish] != null) {
|
|
1111
|
-
debug('%s version %s already exists', name, versionToPublish);
|
|
1136
|
+
debug('%s version %s already exists (upstream)', name, versionToPublish);
|
|
1112
1137
|
throw errorUtils.getConflict();
|
|
1113
1138
|
}
|
|
1114
1139
|
|
|
1115
1140
|
const hasPackageInStorage = await this.hasPackage(name);
|
|
1116
1141
|
if (!hasPackageInStorage) {
|
|
1117
|
-
await this.createNewLocalCachePackage(name,
|
|
1142
|
+
await this.createNewLocalCachePackage(name, username);
|
|
1118
1143
|
successResponseMessage = API_MESSAGE.PKG_CREATED;
|
|
1119
1144
|
} else {
|
|
1145
|
+
await this.checkAllowedToChangePackage(localManifest as Manifest, username);
|
|
1120
1146
|
successResponseMessage = API_MESSAGE.PKG_CHANGED;
|
|
1121
1147
|
}
|
|
1122
1148
|
} catch (err: any) {
|
|
1123
1149
|
debug('error on change or update a package with %o', err.message);
|
|
1124
|
-
logger.error({ err: err.message }, 'error on
|
|
1150
|
+
this.logger.error({ err: err.message }, 'error on publish new version: @{err}');
|
|
1125
1151
|
throw err;
|
|
1126
1152
|
}
|
|
1127
1153
|
|
|
1128
1154
|
// 1. after tarball has been successfully uploaded, we update the version
|
|
1129
1155
|
try {
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1156
|
+
const tarballStats = await this.getTarballStats(versions[versionToPublish], buffer);
|
|
1157
|
+
// Older package managers like npm6 do not send readme content as part of version but include it on root level
|
|
1158
|
+
if (_.isEmpty(versions[versionToPublish].readme)) {
|
|
1159
|
+
versions[versionToPublish].readme =
|
|
1160
|
+
_.isNil(manifest.readme) === false ? String(manifest.readme) : '';
|
|
1161
|
+
}
|
|
1162
|
+
// addVersion will move the readme from the the published version to the root level
|
|
1163
|
+
await this.addVersion(name, versionToPublish, versions[versionToPublish], null, tarballStats);
|
|
1134
1164
|
} catch (err: any) {
|
|
1135
|
-
logger.error({ err: err.message }, 'updated version has failed: @{err}');
|
|
1165
|
+
this.logger.error({ err: err.message }, 'updated version has failed: @{err}');
|
|
1136
1166
|
debug('error on create a version for %o with error %o', name, err.message);
|
|
1137
|
-
// TODO: remove tarball if add version fails
|
|
1138
1167
|
throw err;
|
|
1139
1168
|
}
|
|
1140
1169
|
|
|
@@ -1145,14 +1174,13 @@ class Storage {
|
|
|
1145
1174
|
// 1. add version
|
|
1146
1175
|
// 2. merge versions
|
|
1147
1176
|
// 3. upload tarball
|
|
1148
|
-
//
|
|
1177
|
+
// 4. update once to the storage (easy peasy)
|
|
1149
1178
|
mergedManifest = await this.mergeTagsNext(name, manifest[DIST_TAGS]);
|
|
1150
1179
|
} catch (err: any) {
|
|
1151
|
-
logger.error({ err: err.message }, 'merge version has failed: @{err}');
|
|
1180
|
+
this.logger.error({ err: err.message }, 'merge version has failed: @{err}');
|
|
1152
1181
|
debug('error on create a version for %o with error %o', name, err.message);
|
|
1153
1182
|
// TODO: undo if this fails
|
|
1154
|
-
// 1. remove
|
|
1155
|
-
// 2. remove updated version
|
|
1183
|
+
// 1. remove updated version
|
|
1156
1184
|
throw err;
|
|
1157
1185
|
}
|
|
1158
1186
|
|
|
@@ -1163,11 +1191,14 @@ class Storage {
|
|
|
1163
1191
|
signal: options.signal,
|
|
1164
1192
|
});
|
|
1165
1193
|
} catch (err: any) {
|
|
1166
|
-
logger.error({ err: err.message }, 'upload tarball has failed: @{err}');
|
|
1194
|
+
this.logger.error({ err: err.message }, 'upload tarball has failed: @{err}');
|
|
1195
|
+
// TODO: undo if this fails
|
|
1196
|
+
// 1. remove updated version
|
|
1197
|
+
// 2. remove new dist tags
|
|
1167
1198
|
throw err;
|
|
1168
1199
|
}
|
|
1169
1200
|
|
|
1170
|
-
logger.info(
|
|
1201
|
+
this.logger.info(
|
|
1171
1202
|
{ name, version: versionToPublish },
|
|
1172
1203
|
'package @{name}@@{version} has been published'
|
|
1173
1204
|
);
|
|
@@ -1331,17 +1362,27 @@ class Storage {
|
|
|
1331
1362
|
name: string,
|
|
1332
1363
|
version: string,
|
|
1333
1364
|
metadata: Version,
|
|
1334
|
-
tag: StringValue
|
|
1365
|
+
tag: StringValue,
|
|
1366
|
+
tarballStats: TarballDetails
|
|
1335
1367
|
): Promise<void> {
|
|
1336
1368
|
debug(`add version %s package for %s`, version, name);
|
|
1337
1369
|
await this.updatePackage(name, async (data: Manifest): Promise<Manifest> => {
|
|
1338
1370
|
// keep only one readme per package
|
|
1339
1371
|
data.readme = metadata.readme;
|
|
1340
|
-
debug('%s
|
|
1372
|
+
debug('%s readme mutated', name);
|
|
1341
1373
|
// TODO: lodash remove
|
|
1342
1374
|
metadata = cleanUpReadme(metadata);
|
|
1343
1375
|
metadata.contributors = normalizeContributors(metadata.contributors as Author[]);
|
|
1344
|
-
debug('%s
|
|
1376
|
+
debug('%s contributors normalized', name);
|
|
1377
|
+
|
|
1378
|
+
// Copy current owners to version
|
|
1379
|
+
metadata.maintainers = data.maintainers;
|
|
1380
|
+
|
|
1381
|
+
// Update tarball stats
|
|
1382
|
+
if (metadata.dist) {
|
|
1383
|
+
metadata.dist.fileCount = tarballStats.fileCount;
|
|
1384
|
+
metadata.dist.unpackedSize = tarballStats.unpackedSize;
|
|
1385
|
+
}
|
|
1345
1386
|
|
|
1346
1387
|
// if uploaded tarball has a different shasum, it's very likely that we
|
|
1347
1388
|
// have some kind of error
|
|
@@ -1396,7 +1437,7 @@ class Storage {
|
|
|
1396
1437
|
tagVersion(data, version, tag);
|
|
1397
1438
|
|
|
1398
1439
|
try {
|
|
1399
|
-
debug('%s
|
|
1440
|
+
debug('%s add on database', name);
|
|
1400
1441
|
await this.localStorage.getStoragePlugin().add(name);
|
|
1401
1442
|
this.logger.debug({ name, version }, 'version @{version} added to database for @{name}');
|
|
1402
1443
|
} catch (err: any) {
|
|
@@ -1411,7 +1452,10 @@ class Storage {
|
|
|
1411
1452
|
* @param name name of the package
|
|
1412
1453
|
* @returns
|
|
1413
1454
|
*/
|
|
1414
|
-
private async createNewLocalCachePackage(
|
|
1455
|
+
private async createNewLocalCachePackage(
|
|
1456
|
+
name: string,
|
|
1457
|
+
username: string | undefined
|
|
1458
|
+
): Promise<void> {
|
|
1415
1459
|
const storage: pluginUtils.StorageHandler = this.getPrivatePackageStorage(name);
|
|
1416
1460
|
|
|
1417
1461
|
if (!storage) {
|
|
@@ -1425,10 +1469,16 @@ class Storage {
|
|
|
1425
1469
|
time: {
|
|
1426
1470
|
created: currentTime,
|
|
1427
1471
|
modified: currentTime,
|
|
1428
|
-
[latestVersion]: currentTime,
|
|
1429
1472
|
},
|
|
1430
1473
|
};
|
|
1431
1474
|
|
|
1475
|
+
// Set initial package owner
|
|
1476
|
+
// TODO: Add email of user
|
|
1477
|
+
packageData.maintainers =
|
|
1478
|
+
username && username.length > 0
|
|
1479
|
+
? [{ name: username, email: '' }]
|
|
1480
|
+
: [{ name: DEFAULT_USER, email: '' }];
|
|
1481
|
+
|
|
1432
1482
|
try {
|
|
1433
1483
|
await storage.createPackage(name, packageData);
|
|
1434
1484
|
this.logger.info({ name }, 'created new package @{name}');
|
|
@@ -1454,7 +1504,7 @@ class Storage {
|
|
|
1454
1504
|
private async checkPackageRemote(name: string, uplinksLook: boolean): Promise<Manifest | null> {
|
|
1455
1505
|
try {
|
|
1456
1506
|
// we provide a null manifest, thus the manifest returned will be the remote one
|
|
1457
|
-
const [remoteManifest, upLinksErrors] = await this.
|
|
1507
|
+
const [remoteManifest, upLinksErrors] = await this.syncUplinksMetadata(name, null, {
|
|
1458
1508
|
uplinksLook,
|
|
1459
1509
|
});
|
|
1460
1510
|
|
|
@@ -1567,7 +1617,7 @@ class Storage {
|
|
|
1567
1617
|
// if we can't get the local metadata, we try to get the remote metadata
|
|
1568
1618
|
// if we do to have local metadata, we try to update it with the upstream registry
|
|
1569
1619
|
debug('sync uplinks for %o', name);
|
|
1570
|
-
const [remoteManifest, upLinksErrors] = await this.
|
|
1620
|
+
const [remoteManifest, upLinksErrors] = await this.syncUplinksMetadata(name, data, {
|
|
1571
1621
|
uplinksLook: options.uplinksLook,
|
|
1572
1622
|
retry: options.retry,
|
|
1573
1623
|
remoteAddress: options.requestOptions.remoteAddress,
|
|
@@ -1590,7 +1640,7 @@ class Storage {
|
|
|
1590
1640
|
_attachments: {},
|
|
1591
1641
|
});
|
|
1592
1642
|
|
|
1593
|
-
debug('no
|
|
1643
|
+
debug('no sync uplinks errors %o for %s', upLinksErrors?.length, name);
|
|
1594
1644
|
return [normalizedPkg, upLinksErrors];
|
|
1595
1645
|
}
|
|
1596
1646
|
|
|
@@ -1618,7 +1668,7 @@ class Storage {
|
|
|
1618
1668
|
in that case the request returns empty body and we want ask next on the list if has fresh
|
|
1619
1669
|
updates.
|
|
1620
1670
|
*/
|
|
1621
|
-
public async
|
|
1671
|
+
public async syncUplinksMetadata(
|
|
1622
1672
|
name: string,
|
|
1623
1673
|
localManifest: Manifest | null,
|
|
1624
1674
|
options: Partial<ISyncUplinksOptions> = {}
|
|
@@ -1954,6 +2004,35 @@ class Storage {
|
|
|
1954
2004
|
return cacheManifest;
|
|
1955
2005
|
}
|
|
1956
2006
|
}
|
|
2007
|
+
|
|
2008
|
+
private async getTarballStats(version: Version, buffer: Buffer): Promise<TarballDetails> {
|
|
2009
|
+
if (
|
|
2010
|
+
version.dist == undefined ||
|
|
2011
|
+
version.dist?.fileCount == undefined ||
|
|
2012
|
+
version.dist?.unpackedSize == undefined
|
|
2013
|
+
) {
|
|
2014
|
+
debug('tarball stats not found, calculating');
|
|
2015
|
+
return await getTarballDetails(buffer);
|
|
2016
|
+
} else {
|
|
2017
|
+
debug('tarball stats found');
|
|
2018
|
+
return { fileCount: version.dist.fileCount, unpackedSize: version.dist.unpackedSize };
|
|
2019
|
+
}
|
|
2020
|
+
}
|
|
2021
|
+
|
|
2022
|
+
private async checkAllowedToChangePackage(manifest: Manifest, username: string | undefined) {
|
|
2023
|
+
// Checks to perform if config "publish:check_owners" is true
|
|
2024
|
+
debug('check if user %o is an owner and allowed to change package', username);
|
|
2025
|
+
// if name of owner is not included in list of maintainers, then throw an error
|
|
2026
|
+
if (
|
|
2027
|
+
this.config?.publish?.check_owners === true &&
|
|
2028
|
+
manifest.maintainers &&
|
|
2029
|
+
manifest.maintainers.length > 0 &&
|
|
2030
|
+
!manifest.maintainers.some((maintainer) => maintainer.name === username)
|
|
2031
|
+
) {
|
|
2032
|
+
this.logger.error({ username }, '@{username} is not a maintainer (package owner)');
|
|
2033
|
+
throw Error('only owners are allowed to change package');
|
|
2034
|
+
}
|
|
2035
|
+
}
|
|
1957
2036
|
}
|
|
1958
2037
|
|
|
1959
2038
|
export { Storage };
|