@needmoretruth/nmts-cli 0.36.3 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.ko.md +1 -1
- package/README.md +1 -1
- package/dist/artifact-about.d.ts +1 -1
- package/dist/commands/erase.js +1 -1
- package/dist/commands/organise.d.ts +5 -27
- package/dist/commands/organise.js +33 -193
- package/dist/commands/push.js +1 -1
- package/dist/commands/s3.d.ts +0 -10
- package/dist/commands/s3.js +46 -121
- package/dist/commands/trash.d.ts +0 -9
- package/dist/commands/trash.js +23 -223
- package/dist/drive-edit.d.ts +189 -0
- package/dist/drive-edit.js +541 -0
- package/dist/product.d.ts +1 -1
- package/dist/product.js +1 -1
- package/dist/s3/drive.d.ts +108 -0
- package/dist/s3/drive.js +136 -0
- package/dist/s3/listing.d.ts +8 -1
- package/dist/s3/listing.js +8 -1
- package/dist/s3/server.d.ts +42 -3
- package/dist/s3/server.js +59 -20
- package/dist/s3/sigv4.d.ts +26 -0
- package/dist/s3/sigv4.js +37 -0
- package/dist/s3/xml.d.ts +8 -1
- package/dist/s3/xml.js +13 -4
- package/dist/s3-gateway.d.ts +8 -0
- package/dist/s3-gateway.js +13 -0
- package/docs/commands/create.md +2 -2
- package/docs/commands/env.md +1 -1
- package/docs/commands/extend.md +0 -3
- package/docs/commands/login.md +1 -1
- package/docs/commands/logout.md +1 -1
- package/docs/commands/marks.md +1 -1
- package/docs/commands/mcp.md +1 -1
- package/docs/commands/put.md +2 -2
- package/docs/commands/wallet.md +7 -9
- package/docs/commands/whoami.md +2 -2
- package/package.json +9 -1
package/dist/s3/drive.js
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
// One account behind the gateway: its file list, its reader, and the rule every upload passes.
|
|
2
|
+
//
|
|
3
|
+
// ⛔ ONE IMPLEMENTATION, TWO CALLERS, WHICH IS THE WHOLE REASON THIS FILE EXISTS. `nmts s3` serves
|
|
4
|
+
// the drive of whoever is at this machine; the SDK's gateway serves whichever account a
|
|
5
|
+
// business's resolver hands back. What the two do with an upload -- spool it, ask whether the
|
|
6
|
+
// key already holds exactly these bytes, make the folders above it, store it, forget the cached
|
|
7
|
+
// list -- is the same work, and a second copy of it would be a second place for the same-file
|
|
8
|
+
// rule to be got right. The two differ only in where the account comes from, which is the seam
|
|
9
|
+
// below.
|
|
10
|
+
//
|
|
11
|
+
// ⛔ THE SAME-FILE QUESTION IS ANSWERED HERE AND NOWHERE ELSE. Both ways of uploading -- one PUT,
|
|
12
|
+
// or pieces staged and joined -- end in `storeFile`, so a rule written here cannot disagree with
|
|
13
|
+
// itself; written in the protocol layer it would have to be written twice, once for each, and
|
|
14
|
+
// the two would differ the first time one of them changed.
|
|
15
|
+
import { randomUUID } from "node:crypto";
|
|
16
|
+
import { createWriteStream } from "node:fs";
|
|
17
|
+
import { mkdir as makeDir, rm as removeFile, stat } from "node:fs/promises";
|
|
18
|
+
import { join } from "node:path";
|
|
19
|
+
import { pipeline } from "node:stream/promises";
|
|
20
|
+
import { fetchFile } from "../download.js";
|
|
21
|
+
import { NmtsError } from "../errors.js";
|
|
22
|
+
import { refusalFor, verdictForKey } from "./same-file.js";
|
|
23
|
+
import { createStaging } from "./staging.js";
|
|
24
|
+
/**
|
|
25
|
+
* How long a file list may be reused before it is fetched again.
|
|
26
|
+
*
|
|
27
|
+
* ⛔ THERE IS A CACHE BECAUSE A SYNC IS THOUSANDS OF REQUESTS. Reading the list per request would
|
|
28
|
+
* mean a server round trip and a decryption for each one, so a listing of a large drive would
|
|
29
|
+
* take minutes and cost the account's rate budget. ⚠ It also means a file uploaded from another
|
|
30
|
+
* device can be up to this long in appearing here, which is the trade and is written in the
|
|
31
|
+
* tool's own words when it starts.
|
|
32
|
+
*/
|
|
33
|
+
export const LIST_CACHE_MS = 5_000;
|
|
34
|
+
/**
|
|
35
|
+
* `photos/2026/a.jpg` → the folder to make and the name to store under.
|
|
36
|
+
*
|
|
37
|
+
* A key with no slash lands at the top of the account, which is `undefined` rather than `""`: the
|
|
38
|
+
* two mean the same thing to a person and different things to the upload path.
|
|
39
|
+
*/
|
|
40
|
+
export function placeOf(key) {
|
|
41
|
+
const at = key.lastIndexOf("/");
|
|
42
|
+
if (at < 0)
|
|
43
|
+
return { folder: undefined, name: key };
|
|
44
|
+
const folder = key.slice(0, at);
|
|
45
|
+
return { folder: folder === "" ? undefined : folder, name: key.slice(at + 1) };
|
|
46
|
+
}
|
|
47
|
+
/** The real reader: the stored bytes, opened with this account's key and delivered to the sink. */
|
|
48
|
+
export async function fetchObject(reader, object, sink) {
|
|
49
|
+
const wrapped = object.entry.dekWrapped;
|
|
50
|
+
if (wrapped === undefined)
|
|
51
|
+
throw new NmtsError("That entry has no key in the file list.");
|
|
52
|
+
await fetchFile({
|
|
53
|
+
base: reader.server,
|
|
54
|
+
apiKey: reader.bearer,
|
|
55
|
+
accountCode: reader.code,
|
|
56
|
+
itemId: object.entry.id,
|
|
57
|
+
size: object.size,
|
|
58
|
+
dekWrapped: wrapped,
|
|
59
|
+
contentHashCt: object.entry.contentHashCt,
|
|
60
|
+
chain: reader.chain,
|
|
61
|
+
sink,
|
|
62
|
+
...(reader.read === undefined ? {} : { read: reader.read }),
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
/** One account as the protocol layer sees it: a cached list, a reader, and a writer when allowed. */
|
|
66
|
+
export function createDriveSource(options) {
|
|
67
|
+
const account = options.account;
|
|
68
|
+
const cacheMs = options.listCacheMs ?? LIST_CACHE_MS;
|
|
69
|
+
let cached = [];
|
|
70
|
+
let cachedAt = 0;
|
|
71
|
+
const entries = async () => {
|
|
72
|
+
if (Date.now() - cachedAt < cacheMs)
|
|
73
|
+
return cached;
|
|
74
|
+
cached = await account.readList();
|
|
75
|
+
cachedAt = Date.now();
|
|
76
|
+
return cached;
|
|
77
|
+
};
|
|
78
|
+
/**
|
|
79
|
+
* Store one local file at a drive key, making the folders above it if they are missing.
|
|
80
|
+
*
|
|
81
|
+
* ⭐ IDENTICAL CONTENT IS NOT AN ERROR. Nothing is sent and nothing is charged, and the caller is
|
|
82
|
+
* told the upload finished — because the statement it was making, "that file is at that key",
|
|
83
|
+
* is true. Answering 409 there is what made every backup run fail on every file it had already
|
|
84
|
+
* stored, and a sync tool writes 409 down as a failure.
|
|
85
|
+
*/
|
|
86
|
+
const storeFile = async (key, path) => {
|
|
87
|
+
const { folder, name } = placeOf(key);
|
|
88
|
+
const known = await entries();
|
|
89
|
+
const verdict = await account.withCode((code) => verdictForKey(known, key, code, path));
|
|
90
|
+
if (verdict === "same") {
|
|
91
|
+
options.onAlreadyStored?.(key);
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
if (verdict !== "free")
|
|
95
|
+
throw refusalFor(verdict, key);
|
|
96
|
+
if (folder !== undefined)
|
|
97
|
+
await account.makeFolder(folder);
|
|
98
|
+
await account.store(path, name, folder);
|
|
99
|
+
cachedAt = 0;
|
|
100
|
+
};
|
|
101
|
+
return {
|
|
102
|
+
entries,
|
|
103
|
+
fetch: (object, sink) => account.fetch(object, sink),
|
|
104
|
+
...(options.writable
|
|
105
|
+
? {
|
|
106
|
+
write: {
|
|
107
|
+
// ⛔ THE BODY IS SPOOLED TO A FILE FIRST, 0600, and deleted whatever happens. The
|
|
108
|
+
// upload path reserves storage, cuts parts and seals them from a file, and giving it
|
|
109
|
+
// a socket instead would mean either holding whole uploads in memory or writing a
|
|
110
|
+
// second upload path — and a second upload path is a second place for "what if the
|
|
111
|
+
// reservation succeeds and the part fails" to be got right.
|
|
112
|
+
put: async (key, body, size) => {
|
|
113
|
+
await makeDir(options.stagingRoot, { recursive: true, mode: 0o700 });
|
|
114
|
+
const spool = join(options.stagingRoot, randomUUID());
|
|
115
|
+
try {
|
|
116
|
+
await pipeline(body, createWriteStream(spool, { mode: 0o600 }));
|
|
117
|
+
const written = (await stat(spool)).size;
|
|
118
|
+
if (written !== size) {
|
|
119
|
+
throw new NmtsError(`The upload said ${size} bytes and ${written} arrived. Nothing was stored.`);
|
|
120
|
+
}
|
|
121
|
+
await storeFile(key, spool);
|
|
122
|
+
}
|
|
123
|
+
finally {
|
|
124
|
+
await removeFile(spool, { force: true });
|
|
125
|
+
}
|
|
126
|
+
},
|
|
127
|
+
multipart: options.multipart ?? createStaging(options.stagingRoot, storeFile),
|
|
128
|
+
trash: async (object) => {
|
|
129
|
+
await account.trash(`/${object.key}`);
|
|
130
|
+
cachedAt = 0;
|
|
131
|
+
},
|
|
132
|
+
},
|
|
133
|
+
}
|
|
134
|
+
: {}),
|
|
135
|
+
};
|
|
136
|
+
}
|
package/dist/s3/listing.d.ts
CHANGED
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
import type { ManifestEntry } from "../shared/lib/drive/manifest-codec.ts";
|
|
2
2
|
import type { ObjectRow } from "./xml.ts";
|
|
3
|
-
/**
|
|
3
|
+
/**
|
|
4
|
+
* The bucket `nmts s3` serves. Named for what it is, and not configurable: two names for one drive
|
|
5
|
+
* is worse.
|
|
6
|
+
*
|
|
7
|
+
* ⚠ IT IS THIS COMMAND'S ANSWER, NOT THE SERVER'S RULE. The gateway takes a resolver, because a
|
|
8
|
+
* business running it in front of many of its users' accounts has one bucket per account; what
|
|
9
|
+
* `nmts s3` hands it is a resolver that knows this name and no other.
|
|
10
|
+
*/
|
|
4
11
|
export declare const BUCKET = "drive";
|
|
5
12
|
/** S3's own ceiling, and the default when a client does not ask for one. */
|
|
6
13
|
export declare const MAX_KEYS_LIMIT = 1000;
|
package/dist/s3/listing.js
CHANGED
|
@@ -13,7 +13,14 @@
|
|
|
13
13
|
// prefix. Real S3 has no folders, so an empty one cannot exist there; this drive does have them,
|
|
14
14
|
// and hiding them would make `rclone lsd` describe a drive that is not the one in the browser.
|
|
15
15
|
import { buildIndex, fullPathOf, isLive, KIND_FOLDER } from "../drive-paths.js";
|
|
16
|
-
/**
|
|
16
|
+
/**
|
|
17
|
+
* The bucket `nmts s3` serves. Named for what it is, and not configurable: two names for one drive
|
|
18
|
+
* is worse.
|
|
19
|
+
*
|
|
20
|
+
* ⚠ IT IS THIS COMMAND'S ANSWER, NOT THE SERVER'S RULE. The gateway takes a resolver, because a
|
|
21
|
+
* business running it in front of many of its users' accounts has one bucket per account; what
|
|
22
|
+
* `nmts s3` hands it is a resolver that knows this name and no other.
|
|
23
|
+
*/
|
|
17
24
|
export const BUCKET = "drive";
|
|
18
25
|
/** S3's own ceiling, and the default when a client does not ask for one. */
|
|
19
26
|
export const MAX_KEYS_LIMIT = 1000;
|
package/dist/s3/server.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Server } from "node:http";
|
|
1
|
+
import { type IncomingMessage, type Server, type ServerResponse } from "node:http";
|
|
2
2
|
import type { Readable } from "node:stream";
|
|
3
3
|
import type { PlaintextSink } from "../download-sink.ts";
|
|
4
4
|
import type { ManifestEntry } from "../shared/lib/drive/manifest-codec.ts";
|
|
@@ -46,13 +46,52 @@ export interface DriveWriter {
|
|
|
46
46
|
};
|
|
47
47
|
}
|
|
48
48
|
export interface GatewayOptions {
|
|
49
|
-
|
|
50
|
-
|
|
49
|
+
/**
|
|
50
|
+
* Every pair that may sign a request here, each optionally held to named buckets.
|
|
51
|
+
*
|
|
52
|
+
* ⛔ A LIST RATHER THAN ONE PAIR BECAUSE A BUCKET IS AN ACCOUNT. `nmts s3` makes one pair for one
|
|
53
|
+
* drive; a business serving many of its users' accounts hands each of them a pair of their
|
|
54
|
+
* own, and the restriction on the pair is what stops one customer reading another's bucket.
|
|
55
|
+
*/
|
|
56
|
+
readonly credentials: readonly GatewayCredential[];
|
|
57
|
+
/**
|
|
58
|
+
* Which drive answers to this bucket name, or null when none does.
|
|
59
|
+
*
|
|
60
|
+
* ⛔ THE GATEWAY DOES NOT KNOW WHAT A BUCKET IS. It was one name and one drive for as long as the
|
|
61
|
+
* only caller was the command-line tool; asked by a business's server it is a lookup that
|
|
62
|
+
* server does, and one it may do differently per name. What must not change is that a name
|
|
63
|
+
* this resolver refuses looks exactly like a name the caller may not touch (see below).
|
|
64
|
+
*/
|
|
65
|
+
readonly bucketOf: (name: string) => DriveSource | null | Promise<DriveSource | null>;
|
|
66
|
+
/**
|
|
67
|
+
* The names `ListBuckets` answers with, before the signing pair's own restriction is applied.
|
|
68
|
+
*
|
|
69
|
+
* ⚠ ABSENT IS A REAL ANSWER RATHER THAN A GAP. A gateway in front of a business's own lookup
|
|
70
|
+
* cannot enumerate its customers, so what it can honestly name is what the presented pair is
|
|
71
|
+
* held to — and an unrestricted pair on such a gateway is told nothing, which is true.
|
|
72
|
+
*/
|
|
73
|
+
readonly bucketNames?: () => readonly string[] | Promise<readonly string[]>;
|
|
51
74
|
/** Called with one line whenever a request is answered, so a person can watch what a tool does. */
|
|
52
75
|
readonly log?: (line: string) => void;
|
|
53
76
|
/** Passed in so a test can hold the clock still. */
|
|
54
77
|
readonly now?: () => number;
|
|
78
|
+
/**
|
|
79
|
+
* The sentence a write gets from a read-only drive. `nmts s3` says what a person runs on this
|
|
80
|
+
* machine to allow spending; a gateway somebody else runs has a different way in, and says its own.
|
|
81
|
+
*/
|
|
82
|
+
readonly readOnlyBecause?: string | undefined;
|
|
55
83
|
}
|
|
56
84
|
/** A random pair, made fresh every time the gateway starts and stored nowhere. */
|
|
57
85
|
export declare function newCredential(): GatewayCredential;
|
|
86
|
+
/** A plain Node request handler, so this can be mounted in somebody else's server. */
|
|
87
|
+
export type GatewayHandler = (req: IncomingMessage, res: ServerResponse) => void;
|
|
88
|
+
/**
|
|
89
|
+
* The gateway as a handler, which is the form that listens to nothing.
|
|
90
|
+
*
|
|
91
|
+
* ⛔ SEPARATE FROM `createGateway` BECAUSE WHO LISTENS IS NOT THIS FILE'S DECISION. The
|
|
92
|
+
* command-line tool binds loopback and says why at the top of this file; a business mounting
|
|
93
|
+
* this behind its own TLS has already made that decision, and a library that opened a socket of
|
|
94
|
+
* its own would be making it again, differently.
|
|
95
|
+
*/
|
|
96
|
+
export declare function gatewayHandler(options: GatewayOptions): GatewayHandler;
|
|
58
97
|
export declare function createGateway(options: GatewayOptions): Server;
|
package/dist/s3/server.js
CHANGED
|
@@ -12,14 +12,23 @@
|
|
|
12
12
|
// that never happened.
|
|
13
13
|
import { createServer } from "node:http";
|
|
14
14
|
import { randomBytes } from "node:crypto";
|
|
15
|
-
import {
|
|
15
|
+
import { listObjects, objectsOf, folderPrefixesOf, MAX_KEYS_LIMIT } from "./listing.js";
|
|
16
16
|
import { handleMultipart, isMultipartRequest } from "./multipart.js";
|
|
17
17
|
import { responseSink } from "./response-sink.js";
|
|
18
18
|
import { isKeyConflict } from "./same-file.js";
|
|
19
|
-
import { STREAMING_PAYLOAD, STREAMING_PAYLOAD_TRAILER,
|
|
19
|
+
import { STREAMING_PAYLOAD, STREAMING_PAYLOAD_TRAILER, verifyAgainst, } from "./sigv4.js";
|
|
20
20
|
import { errorXml, listBucketsXml, listObjectsXml } from "./xml.js";
|
|
21
21
|
/** Where the drive is served. Loopback, always — see the note above. */
|
|
22
22
|
export const BIND_ADDRESS = "127.0.0.1";
|
|
23
|
+
/**
|
|
24
|
+
* What a caller signed with a pair it may not use here.
|
|
25
|
+
*
|
|
26
|
+
* ⛔ THE SAME ANSWER WHETHER OR NOT THE BUCKET EXISTS, which is why the restriction is checked
|
|
27
|
+
* before the resolver is asked. Answering `NoSuchBucket` for a name the caller may not touch
|
|
28
|
+
* would turn this gateway into a way of asking "does this business have a customer called…",
|
|
29
|
+
* one guess at a time.
|
|
30
|
+
*/
|
|
31
|
+
const NOT_YOURS = "That access key may not use that bucket.";
|
|
23
32
|
/** A random pair, made fresh every time the gateway starts and stored nowhere. */
|
|
24
33
|
export function newCredential() {
|
|
25
34
|
const letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
|
|
@@ -46,12 +55,25 @@ function headerOf(req, name) {
|
|
|
46
55
|
const raw = req.headers[name];
|
|
47
56
|
return Array.isArray(raw) ? raw.join(",") : raw;
|
|
48
57
|
}
|
|
49
|
-
/** The one sentence a write gets when this machine has not agreed to spending. */
|
|
50
|
-
function
|
|
58
|
+
/** The one sentence a write gets from `nmts s3` when this machine has not agreed to spending. */
|
|
59
|
+
function readOnlyOnThisMachine() {
|
|
51
60
|
return ("This gateway is read only. Uploading spends credits, and this machine has not agreed to " +
|
|
52
61
|
"spending — `nmts consent grant spend`, run by the person whose account this is, is what " +
|
|
53
62
|
"changes that. Nothing was written.");
|
|
54
63
|
}
|
|
64
|
+
/** Whether the pair that signed is allowed anywhere near this bucket name. */
|
|
65
|
+
function mayTouch(credential, bucket) {
|
|
66
|
+
const only = credential.buckets;
|
|
67
|
+
return only === undefined || only.includes(bucket);
|
|
68
|
+
}
|
|
69
|
+
/** What `ListBuckets` says: what the gateway can name, narrowed to what this pair may touch. */
|
|
70
|
+
async function bucketsFor(options, credential) {
|
|
71
|
+
const only = credential.buckets;
|
|
72
|
+
if (options.bucketNames === undefined)
|
|
73
|
+
return only ?? [];
|
|
74
|
+
const named = await options.bucketNames();
|
|
75
|
+
return only === undefined ? named : named.filter((name) => only.includes(name));
|
|
76
|
+
}
|
|
55
77
|
function objectHeaders(object) {
|
|
56
78
|
return {
|
|
57
79
|
"content-type": "application/octet-stream",
|
|
@@ -66,20 +88,26 @@ async function handle(req, res, options) {
|
|
|
66
88
|
const pathname = at < 0 ? url : url.slice(0, at);
|
|
67
89
|
const query = new URLSearchParams(at < 0 ? "" : url.slice(at + 1));
|
|
68
90
|
const method = (req.method ?? "GET").toUpperCase();
|
|
69
|
-
const verdict =
|
|
91
|
+
const verdict = verifyAgainst({ method, url, headers: req.headers }, options.credentials, options.now?.() ?? Date.now());
|
|
70
92
|
if (!verdict.ok) {
|
|
71
|
-
fail(res,
|
|
93
|
+
fail(res, 403, verdict.code, verdict.message, pathname);
|
|
72
94
|
return;
|
|
73
95
|
}
|
|
96
|
+
const credential = verdict.credential;
|
|
74
97
|
const { bucket, key } = splitPath(pathname);
|
|
75
98
|
if (pathname === "/" && (method === "GET" || method === "HEAD")) {
|
|
76
|
-
const body = listBucketsXml(
|
|
99
|
+
const body = listBucketsXml(await bucketsFor(options, credential), new Date(0).toISOString());
|
|
77
100
|
res.writeHead(200, { "content-type": "application/xml", "content-length": String(Buffer.byteLength(body)) });
|
|
78
101
|
res.end(method === "HEAD" ? undefined : body);
|
|
79
102
|
return;
|
|
80
103
|
}
|
|
81
|
-
if (bucket
|
|
82
|
-
fail(res,
|
|
104
|
+
if (!mayTouch(credential, bucket)) {
|
|
105
|
+
fail(res, 403, "AccessDenied", NOT_YOURS, pathname);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
const source = await options.bucketOf(bucket);
|
|
109
|
+
if (source === null) {
|
|
110
|
+
fail(res, 404, "NoSuchBucket", `No bucket named ${bucket} is served here.`, pathname);
|
|
83
111
|
return;
|
|
84
112
|
}
|
|
85
113
|
// ⛔ MEASURED, NOT GUESSED: rclone's first act when copying a file is to create the bucket, and a
|
|
@@ -90,7 +118,7 @@ async function handle(req, res, options) {
|
|
|
90
118
|
res.end();
|
|
91
119
|
return;
|
|
92
120
|
}
|
|
93
|
-
const entries = await
|
|
121
|
+
const entries = await source.entries();
|
|
94
122
|
if (key === "" && (method === "GET" || method === "HEAD")) {
|
|
95
123
|
const objects = objectsOf(entries);
|
|
96
124
|
const listing = listObjects(objects, folderPrefixesOf(entries), {
|
|
@@ -100,7 +128,7 @@ async function handle(req, res, options) {
|
|
|
100
128
|
after: query.get("continuation-token") ?? query.get("start-after") ?? query.get("marker"),
|
|
101
129
|
});
|
|
102
130
|
const body = listObjectsXml({
|
|
103
|
-
bucket
|
|
131
|
+
bucket,
|
|
104
132
|
prefix: query.get("prefix") ?? "",
|
|
105
133
|
delimiter: query.get("delimiter") ?? "",
|
|
106
134
|
maxKeys: Number(query.get("max-keys") ?? MAX_KEYS_LIMIT) || MAX_KEYS_LIMIT,
|
|
@@ -134,7 +162,7 @@ async function handle(req, res, options) {
|
|
|
134
162
|
}
|
|
135
163
|
const sink = responseSink(res, { headers: objectHeaders(object) });
|
|
136
164
|
try {
|
|
137
|
-
await
|
|
165
|
+
await source.fetch(object, sink);
|
|
138
166
|
options.log?.(`GET ${key} → ${object.size} bytes`);
|
|
139
167
|
}
|
|
140
168
|
catch (error) {
|
|
@@ -146,7 +174,7 @@ async function handle(req, res, options) {
|
|
|
146
174
|
}
|
|
147
175
|
return;
|
|
148
176
|
}
|
|
149
|
-
const writer =
|
|
177
|
+
const writer = source.write;
|
|
150
178
|
// ⛔ WHETHER A TAKEN KEY IS A CONFLICT IS NOT DECIDED HERE.
|
|
151
179
|
// It used to be, on the strength of the NAME alone, and both upload paths carried their own
|
|
152
180
|
// copy of that check. The question is now about CONTENT — is the file arriving the file
|
|
@@ -160,13 +188,13 @@ async function handle(req, res, options) {
|
|
|
160
188
|
};
|
|
161
189
|
if (isMultipartRequest(method, query) && key !== "") {
|
|
162
190
|
if (writer === undefined) {
|
|
163
|
-
fail(res, 501, "NotImplemented", readOnlyBecause(), pathname);
|
|
191
|
+
fail(res, 501, "NotImplemented", options.readOnlyBecause ?? readOnlyOnThisMachine(), pathname);
|
|
164
192
|
return;
|
|
165
193
|
}
|
|
166
194
|
const handled = await handleMultipart({
|
|
167
195
|
req,
|
|
168
196
|
res,
|
|
169
|
-
bucket
|
|
197
|
+
bucket,
|
|
170
198
|
key,
|
|
171
199
|
method,
|
|
172
200
|
query,
|
|
@@ -180,7 +208,7 @@ async function handle(req, res, options) {
|
|
|
180
208
|
}
|
|
181
209
|
if (method === "PUT" && key !== "") {
|
|
182
210
|
if (writer === undefined) {
|
|
183
|
-
fail(res, 501, "NotImplemented", readOnlyBecause(), pathname);
|
|
211
|
+
fail(res, 501, "NotImplemented", options.readOnlyBecause ?? readOnlyOnThisMachine(), pathname);
|
|
184
212
|
return;
|
|
185
213
|
}
|
|
186
214
|
const declared = headerOf(req, "x-amz-content-sha256");
|
|
@@ -213,7 +241,7 @@ async function handle(req, res, options) {
|
|
|
213
241
|
}
|
|
214
242
|
if (method === "DELETE" && key !== "") {
|
|
215
243
|
if (writer === undefined) {
|
|
216
|
-
fail(res, 501, "NotImplemented", readOnlyBecause(), pathname);
|
|
244
|
+
fail(res, 501, "NotImplemented", options.readOnlyBecause ?? readOnlyOnThisMachine(), pathname);
|
|
217
245
|
return;
|
|
218
246
|
}
|
|
219
247
|
const object = objectsOf(entries).find((o) => o.key === key);
|
|
@@ -238,8 +266,16 @@ async function handle(req, res, options) {
|
|
|
238
266
|
}
|
|
239
267
|
fail(res, 501, "NotImplemented", `This gateway does not answer ${method} on that address.`, pathname);
|
|
240
268
|
}
|
|
241
|
-
|
|
242
|
-
|
|
269
|
+
/**
|
|
270
|
+
* The gateway as a handler, which is the form that listens to nothing.
|
|
271
|
+
*
|
|
272
|
+
* ⛔ SEPARATE FROM `createGateway` BECAUSE WHO LISTENS IS NOT THIS FILE'S DECISION. The
|
|
273
|
+
* command-line tool binds loopback and says why at the top of this file; a business mounting
|
|
274
|
+
* this behind its own TLS has already made that decision, and a library that opened a socket of
|
|
275
|
+
* its own would be making it again, differently.
|
|
276
|
+
*/
|
|
277
|
+
export function gatewayHandler(options) {
|
|
278
|
+
return (req, res) => {
|
|
243
279
|
handle(req, res, options).catch((error) => {
|
|
244
280
|
if (!res.headersSent) {
|
|
245
281
|
fail(res, 500, "InternalError", error instanceof Error ? error.message : String(error), req.url ?? "/");
|
|
@@ -248,5 +284,8 @@ export function createGateway(options) {
|
|
|
248
284
|
res.destroy();
|
|
249
285
|
}
|
|
250
286
|
});
|
|
251
|
-
}
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
export function createGateway(options) {
|
|
290
|
+
return createServer(gatewayHandler(options));
|
|
252
291
|
}
|
package/dist/s3/sigv4.d.ts
CHANGED
|
@@ -13,6 +13,14 @@ export interface IncomingRequest {
|
|
|
13
13
|
export interface GatewayCredential {
|
|
14
14
|
readonly accessKeyId: string;
|
|
15
15
|
readonly secretAccessKey: string;
|
|
16
|
+
/**
|
|
17
|
+
* The only buckets this pair may touch. Absent means every bucket the gateway serves.
|
|
18
|
+
*
|
|
19
|
+
* ⛔ IT IS WHAT KEEPS ONE CUSTOMER'S KEY OFF ANOTHER CUSTOMER'S BUCKET. A gateway in front of
|
|
20
|
+
* many accounts hands each caller its own pair, and without this every pair would open every
|
|
21
|
+
* account the resolver knows.
|
|
22
|
+
*/
|
|
23
|
+
readonly buckets?: readonly string[] | undefined;
|
|
16
24
|
}
|
|
17
25
|
export type Verified = {
|
|
18
26
|
readonly ok: true;
|
|
@@ -22,6 +30,16 @@ export type Verified = {
|
|
|
22
30
|
readonly code: string;
|
|
23
31
|
readonly message: string;
|
|
24
32
|
};
|
|
33
|
+
/** What `verifyAgainst` answers: the same verdict, plus which of the pairs signed. */
|
|
34
|
+
export type VerifiedAgainst = {
|
|
35
|
+
readonly ok: true;
|
|
36
|
+
readonly payloadHash: string;
|
|
37
|
+
readonly credential: GatewayCredential;
|
|
38
|
+
} | {
|
|
39
|
+
readonly ok: false;
|
|
40
|
+
readonly code: string;
|
|
41
|
+
readonly message: string;
|
|
42
|
+
};
|
|
25
43
|
/** `k=v&k2=v2` in the order AWS wants: encoded, sorted by key and then by value. */
|
|
26
44
|
export declare function canonicalQuery(rawQuery: string): string;
|
|
27
45
|
interface AuthorizationParts {
|
|
@@ -43,4 +61,12 @@ export declare function amzDateToMs(stamp: string | undefined): number | null;
|
|
|
43
61
|
* skew rule at all, and that rule is the one that stops a captured request being replayed tomorrow.
|
|
44
62
|
*/
|
|
45
63
|
export declare function verifySignature(request: IncomingRequest, credential: GatewayCredential, now: number): Verified;
|
|
64
|
+
/**
|
|
65
|
+
* The whole check, against every pair a gateway answers to: which one signed, and whether it did.
|
|
66
|
+
*
|
|
67
|
+
* ⚠ THE THREE REFUSALS ARE DIFFERENT ON PURPOSE. "No authorization header at all", "a key this
|
|
68
|
+
* gateway does not have" and "a signature that does not hold" are three different things for
|
|
69
|
+
* whoever is reading a client's logs, and none of them says anything about what is in the drive.
|
|
70
|
+
*/
|
|
71
|
+
export declare function verifyAgainst(request: IncomingRequest, credentials: readonly GatewayCredential[], now: number): VerifiedAgainst;
|
|
46
72
|
export {};
|
package/dist/s3/sigv4.js
CHANGED
|
@@ -21,6 +21,7 @@ export const MAX_CLOCK_SKEW_MS = 15 * 60 * 1000;
|
|
|
21
21
|
export const UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD";
|
|
22
22
|
export const STREAMING_PAYLOAD = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
|
|
23
23
|
export const STREAMING_PAYLOAD_TRAILER = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER";
|
|
24
|
+
/** One refusal, shaped so it satisfies both verdict types — neither of which has an `ok: true`. */
|
|
24
25
|
function refuse(code, message) {
|
|
25
26
|
return { ok: false, code, message };
|
|
26
27
|
}
|
|
@@ -166,3 +167,39 @@ export function verifySignature(request, credential, now) {
|
|
|
166
167
|
}
|
|
167
168
|
return { ok: true, payloadHash };
|
|
168
169
|
}
|
|
170
|
+
/**
|
|
171
|
+
* Which of a gateway's pairs the request named, without telling the clock how many there are.
|
|
172
|
+
*
|
|
173
|
+
* ⛔ EVERY PAIR IS COMPARED AND THE LOOP DOES NOT STOP EARLY. An access key id is not a secret --
|
|
174
|
+
* it travels in the header in the clear -- but a scan that returned at the first match would
|
|
175
|
+
* take a length of time that says WHERE in the list a key sits, and that is a fact about the
|
|
176
|
+
* gateway's customers rather than about the request.
|
|
177
|
+
*/
|
|
178
|
+
function named(credentials, accessKeyId) {
|
|
179
|
+
const wanted = Buffer.from(accessKeyId, "utf8");
|
|
180
|
+
let found = null;
|
|
181
|
+
for (const candidate of credentials) {
|
|
182
|
+
const id = Buffer.from(candidate.accessKeyId, "utf8");
|
|
183
|
+
if (id.length === wanted.length && timingSafeEqual(id, wanted))
|
|
184
|
+
found = candidate;
|
|
185
|
+
}
|
|
186
|
+
return found;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* The whole check, against every pair a gateway answers to: which one signed, and whether it did.
|
|
190
|
+
*
|
|
191
|
+
* ⚠ THE THREE REFUSALS ARE DIFFERENT ON PURPOSE. "No authorization header at all", "a key this
|
|
192
|
+
* gateway does not have" and "a signature that does not hold" are three different things for
|
|
193
|
+
* whoever is reading a client's logs, and none of them says anything about what is in the drive.
|
|
194
|
+
*/
|
|
195
|
+
export function verifyAgainst(request, credentials, now) {
|
|
196
|
+
const auth = parseAuthorization(headerValue(request.headers, "authorization"));
|
|
197
|
+
if (auth === null)
|
|
198
|
+
return refuse("AccessDenied", "no AWS Signature Version 4 authorization header");
|
|
199
|
+
const credential = named(credentials, auth.accessKeyId);
|
|
200
|
+
if (credential === null) {
|
|
201
|
+
return refuse("InvalidAccessKeyId", "that access key is not one this gateway answers to");
|
|
202
|
+
}
|
|
203
|
+
const verdict = verifySignature(request, credential, now);
|
|
204
|
+
return verdict.ok ? { ok: true, payloadHash: verdict.payloadHash, credential } : verdict;
|
|
205
|
+
}
|
package/dist/s3/xml.d.ts
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
/** The five characters XML cannot carry raw. */
|
|
2
2
|
export declare function escapeXml(value: string): string;
|
|
3
3
|
export declare function errorXml(code: string, message: string, resource: string): string;
|
|
4
|
-
|
|
4
|
+
/**
|
|
5
|
+
* The answer to `ListBuckets`.
|
|
6
|
+
*
|
|
7
|
+
* ⚠ AN EMPTY LIST IS A LEGAL ANSWER AND EVERY CLIENT HANDLES IT. A gateway in front of a
|
|
8
|
+
* business's own lookup cannot enumerate that business's customers, and naming none is the true
|
|
9
|
+
* answer there — the caller reaches its own bucket by asking for it by name.
|
|
10
|
+
*/
|
|
11
|
+
export declare function listBucketsXml(buckets: readonly string[], createdAt: string): string;
|
|
5
12
|
export declare function initiateUploadXml(bucket: string, key: string, uploadId: string): string;
|
|
6
13
|
export declare function completeUploadXml(bucket: string, key: string, etag: string): string;
|
|
7
14
|
export interface ObjectRow {
|
package/dist/s3/xml.js
CHANGED
|
@@ -23,11 +23,20 @@ export function errorXml(code, message, resource) {
|
|
|
23
23
|
return (`${HEAD}<Error><Code>${escapeXml(code)}</Code><Message>${escapeXml(message)}</Message>` +
|
|
24
24
|
`<Resource>${escapeXml(resource)}</Resource></Error>`);
|
|
25
25
|
}
|
|
26
|
-
|
|
26
|
+
/**
|
|
27
|
+
* The answer to `ListBuckets`.
|
|
28
|
+
*
|
|
29
|
+
* ⚠ AN EMPTY LIST IS A LEGAL ANSWER AND EVERY CLIENT HANDLES IT. A gateway in front of a
|
|
30
|
+
* business's own lookup cannot enumerate that business's customers, and naming none is the true
|
|
31
|
+
* answer there — the caller reaches its own bucket by asking for it by name.
|
|
32
|
+
*/
|
|
33
|
+
export function listBucketsXml(buckets, createdAt) {
|
|
34
|
+
const rows = buckets
|
|
35
|
+
.map((bucket) => `<Bucket><Name>${escapeXml(bucket)}</Name>` +
|
|
36
|
+
`<CreationDate>${escapeXml(createdAt)}</CreationDate></Bucket>`)
|
|
37
|
+
.join("");
|
|
27
38
|
return (`${HEAD}<ListAllMyBucketsResult xmlns="${NS}"><Owner><ID>nmts</ID>` +
|
|
28
|
-
`<DisplayName>nmts</DisplayName></Owner><Buckets
|
|
29
|
-
`<Name>${escapeXml(bucket)}</Name><CreationDate>${escapeXml(createdAt)}</CreationDate>` +
|
|
30
|
-
`</Bucket></Buckets></ListAllMyBucketsResult>`);
|
|
39
|
+
`<DisplayName>nmts</DisplayName></Owner><Buckets>${rows}</Buckets></ListAllMyBucketsResult>`);
|
|
31
40
|
}
|
|
32
41
|
export function initiateUploadXml(bucket, key, uploadId) {
|
|
33
42
|
return (`${HEAD}<InitiateMultipartUploadResult xmlns="${NS}">` +
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { createGateway, gatewayHandler } from "./s3/server.ts";
|
|
2
|
+
export type { DriveSource, DriveWriter, GatewayHandler, GatewayOptions } from "./s3/server.ts";
|
|
3
|
+
export { createDriveSource, fetchObject, placeOf, LIST_CACHE_MS } from "./s3/drive.ts";
|
|
4
|
+
export type { DriveAccount, DriveSourceOptions, ObjectReader } from "./s3/drive.ts";
|
|
5
|
+
export { createStaging } from "./s3/staging.ts";
|
|
6
|
+
export type { Staging, StoreFile } from "./s3/staging.ts";
|
|
7
|
+
export type { GatewayCredential } from "./s3/sigv4.ts";
|
|
8
|
+
export type { DriveObject } from "./s3/listing.ts";
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// The S3 gateway as a library: the server `nmts s3` runs, for somebody else's infrastructure.
|
|
2
|
+
//
|
|
3
|
+
// ⛔ ONE IMPLEMENTATION, AND THIS IS THE DOOR TO IT. A business putting NMTS behind its own storage
|
|
4
|
+
// adapters needs the protocol this package already speaks, in its own process, in front of
|
|
5
|
+
// whichever of its customers' accounts a request names. A copy of the server in the SDK would
|
|
6
|
+
// be a second place for a signature check, a listing and a multipart upload to be got right,
|
|
7
|
+
// and the copy nobody re-reads is the one that quietly disagrees.
|
|
8
|
+
//
|
|
9
|
+
// ⚠ WHAT IS NOT HERE: the bucket name `nmts s3` uses, the address it binds, and the pair it prints
|
|
10
|
+
// when it starts. Those are that command's answers to questions a business answers for itself.
|
|
11
|
+
export { createGateway, gatewayHandler } from "./s3/server.js";
|
|
12
|
+
export { createDriveSource, fetchObject, placeOf, LIST_CACHE_MS } from "./s3/drive.js";
|
|
13
|
+
export { createStaging } from "./s3/staging.js";
|
package/docs/commands/create.md
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
Commands: create
|
|
4
4
|
Tiers: create=high
|
|
5
5
|
|
|
6
|
-
Signs in with one account's key and creates another, printing the new
|
|
7
|
-
print it again, because the server stores a one-way verifier and never the NMTS key. This is how a
|
|
6
|
+
Signs in with one account's key and creates another, printing the new NMTS key once — nothing
|
|
7
|
+
can print it again, because the server stores a one-way verifier and never the NMTS key. This is how a
|
|
8
8
|
service that keeps its customers' files in NMTS gives each customer a drive; the first account of
|
|
9
9
|
all has to be made in a browser. It needs a key with `files:write` and a live human check behind
|
|
10
10
|
it, and the server allows two a day and five a week per key.
|
package/docs/commands/env.md
CHANGED
|
@@ -8,7 +8,7 @@ gives the same thing to parse. It reports the operating system; whether this is
|
|
|
8
8
|
container and whether root here is root on the host; whether a file written here can be kept
|
|
9
9
|
private (measured, not guessed); whether there is a terminal and whether a browser could be
|
|
10
10
|
opened; whether an NMTS key and an API key were found and where each came from; if the stored
|
|
11
|
-
|
|
11
|
+
NMTS key is sealed, whether a passphrase is actually reachable; which agent left a marker here; and
|
|
12
12
|
what the version check last found. The `advice` it returns is written to be repeated to the person
|
|
13
13
|
as-is — do that when something in it is a `warn`.
|
|
14
14
|
|
package/docs/commands/extend.md
CHANGED
|
@@ -15,9 +15,6 @@ nmts extend notes/report.pdf --epochs 4 # how many epochs to add (default 2)
|
|
|
15
15
|
If the account holds a standing share (`nmts tip`), that share of the WAL just paid goes to the
|
|
16
16
|
developer right after the extension, without a question.
|
|
17
17
|
|
|
18
|
-
```sh
|
|
19
|
-
```
|
|
20
|
-
|
|
21
18
|
Both forms print the price in WAL, the chain fee in SUI measured by a dry run of the exact
|
|
22
19
|
transaction (`null` in `--json` when it could not be measured, never 0), and what the wallet
|
|
23
20
|
holds. A wallet short of either exits 4 with the two numbers before anything else; a balance the
|
package/docs/commands/login.md
CHANGED
|
@@ -7,7 +7,7 @@ Tiers: login=none · login.plain=high(unsafe-code-storage) · login.env=high(pla
|
|
|
7
7
|
(`NMTS_API_KEY_FILE`, `NMTS_API_KEY`) with the server, and stores that too. It prints the key's
|
|
8
8
|
public handle and never the key. It does not replace a stored key unless the run says so.
|
|
9
9
|
|
|
10
|
-
Every later command needs the passphrase, from `NMTS_PASSPHRASE` or a terminal. A sealed
|
|
10
|
+
Every later command needs the passphrase, from `NMTS_PASSPHRASE` or a terminal. A sealed key with
|
|
11
11
|
no passphrase in reach is not a usable credential; `nmts env` says which case this machine is in.
|
|
12
12
|
|
|
13
13
|
Two other shapes exist and both are locked until a person opens them once, at a terminal:
|
package/docs/commands/logout.md
CHANGED
|
@@ -5,7 +5,7 @@ Tiers: logout=none
|
|
|
5
5
|
|
|
6
6
|
Removes the NMTS key and API key this tool stored on this machine. Nothing on the server
|
|
7
7
|
changes: the key stays valid until revoked (`nmts key revoke`, or the account screen), and the
|
|
8
|
-
account is untouched.
|
|
8
|
+
account is untouched. An NMTS key that came from an environment variable or a file is not touched
|
|
9
9
|
either — this only forgets what `nmts login` wrote.
|
|
10
10
|
|
|
11
11
|
Run it when a machine is handed over or a task is finished on a machine the person does not keep.
|
package/docs/commands/marks.md
CHANGED
|
@@ -5,5 +5,5 @@ Tiers: star=none · unstar=none · pin=none · unpin=none · label=none · unlab
|
|
|
5
5
|
|
|
6
6
|
Free and reversible; nothing here asks. `star <files>` gathers files in favourites, `pin <files>`
|
|
7
7
|
holds them at the top of their folder, and `label <name> <files>` attaches a word you choose; the
|
|
8
|
-
`un-` forms take each off. A label exists while a file
|
|
8
|
+
`un-` forms take each off. A label exists while a file has it. `label --rename <old> <new>`
|
|
9
9
|
renames a label on every file that carries it; `unlabel <name> --all` takes it off all of them.
|
package/docs/commands/mcp.md
CHANGED
|
@@ -28,6 +28,6 @@ codex mcp add nmts -- nmts mcp --out /where/files/should/land
|
|
|
28
28
|
opencode mcp add nmts -- nmts mcp --out /where/files/should/land
|
|
29
29
|
```
|
|
30
30
|
|
|
31
|
-
A sealed stored
|
|
31
|
+
A sealed stored NMTS key is opened once, at startup; `nmts mcp` never prompts, so a sealed key with
|
|
32
32
|
no `NMTS_PASSPHRASE` exits 3 at startup rather than hang. Arguments are checked against what each
|
|
33
33
|
tool declares — `"dry_run": "true"` is a refusal, not an upload.
|