@needmoretruth/nmts-cli 0.38.0 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +17 -16
- package/CHANGELOG.md +17 -0
- package/README.ko.md +1 -1
- package/README.md +1 -1
- package/dist/artifact-about.d.ts +1 -1
- package/dist/commands/erase.js +12 -107
- package/dist/drive-edit/errors.d.ts +44 -0
- package/dist/drive-edit/errors.js +65 -0
- package/dist/drive-edit/folders.d.ts +29 -0
- package/dist/drive-edit/folders.js +98 -0
- package/dist/drive-edit/move.d.ts +66 -0
- package/dist/drive-edit/move.js +119 -0
- package/dist/drive-edit/trash.d.ts +53 -0
- package/dist/drive-edit/trash.js +190 -0
- package/dist/drive-edit/tree.d.ts +15 -0
- package/dist/drive-edit/tree.js +77 -0
- package/dist/drive-edit.d.ts +9 -189
- package/dist/drive-edit.js +9 -527
- package/dist/drive-erase.d.ts +83 -0
- package/dist/drive-erase.js +161 -0
- package/dist/platform-sign.d.ts +9 -1
- package/dist/platform-sign.js +9 -1
- package/dist/product.d.ts +1 -1
- package/dist/product.js +1 -1
- package/dist/s3/contract.d.ts +80 -0
- package/dist/s3/contract.js +3 -0
- package/dist/s3/routes.d.ts +4 -0
- package/dist/s3/routes.js +249 -0
- package/dist/s3/server.d.ts +3 -80
- package/dist/s3/server.js +4 -247
- package/package.json +5 -1
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
// Erasing files for good, without a terminal: the targets resolved, the treasury's storage let go,
|
|
2
|
+
// the server's records destroyed and the entries taken out of the sealed list — decided, done and
|
|
3
|
+
// handed back rather than printed.
|
|
4
|
+
//
|
|
5
|
+
// ⛔ ONE IMPLEMENTATION, TWO CALLERS, WHICH IS THE WHOLE REASON THIS FILE EXISTS — the argument
|
|
6
|
+
// `drive-edit.ts` makes one verb family over. `commands/erase.ts` is a terminal's shape: it
|
|
7
|
+
// prints the list, asks for the typed sentence and answers an exit code. The SDK is somebody
|
|
8
|
+
// else's program and needs the same act with none of those. A second implementation of "what
|
|
9
|
+
// order do these three destructions happen in" would be a second place for the rules below to
|
|
10
|
+
// be got right, and the copy nobody re-reads is the one that quietly disagrees — about the one
|
|
11
|
+
// verb in this package that nothing can undo.
|
|
12
|
+
//
|
|
13
|
+
// ⛔ NOTHING HERE WRITES TO A STREAM, PICKS AN EXIT CODE OR ASKS ANYBODY ANYTHING. The confirmation
|
|
14
|
+
// is the caller's: a terminal types a sentence, a program passes one. This file is reached only
|
|
15
|
+
// after somebody decided, and it destroys what it is given.
|
|
16
|
+
//
|
|
17
|
+
// ⛔ THE SERVER GOES FIRST AND THE LIST LAST, which is the order the trash keeps and for the same
|
|
18
|
+
// reason: a row erased before its list entry leaves a file the person can see and never open.
|
|
19
|
+
// A release that FAILS stops the run before a single row is touched — nothing is erased behind
|
|
20
|
+
// one. A release the server REFUSES ("that storage was bought by the wallet, not by credits")
|
|
21
|
+
// is an answer rather than a failure: it is reported on that file and the erase goes on.
|
|
22
|
+
//
|
|
23
|
+
// ⛔ THE PROOF IS THE CALLER'S TO BUILD AND IS NOT DERIVED HERE. Both doors ask for the account
|
|
24
|
+
// code's own proof, and WHETHER this run may make one is a policy question with two different
|
|
25
|
+
// answers — the command line asks for an agreement first (`account-proof.ts`), a library caller
|
|
26
|
+
// has already made that decision by calling. So it arrives as a value. Nothing here reaches for
|
|
27
|
+
// `node:`: the SDK's browser entry bundles what this exports.
|
|
28
|
+
//
|
|
29
|
+
// ⚠ THE PIECES IT BORROWS COME FROM `drive-edit/` DIRECTLY, the way that folder's own files borrow
|
|
30
|
+
// from each other. This is a sibling of that family rather than an outside caller.
|
|
31
|
+
import { request, ServerError } from "./api.js";
|
|
32
|
+
import { DriveEditError, resolving } from "./drive-edit/errors.js";
|
|
33
|
+
import { filesUnder, uniqueById } from "./drive-edit/tree.js";
|
|
34
|
+
import { buildIndex, fullPathOf, KIND_FILE } from "./drive-paths.js";
|
|
35
|
+
import { NmtsError } from "./errors.js";
|
|
36
|
+
import { readFileList } from "./manifest.js";
|
|
37
|
+
import { applyToList, batchTargets } from "./manifest-write.js";
|
|
38
|
+
import { BINARY_NAME } from "./product.js";
|
|
39
|
+
/** The server takes at most this many ids in one erase (`ERASE_BATCH_MAX`). */
|
|
40
|
+
const BATCH = 200;
|
|
41
|
+
/** The refusal a no-deposit release gets when the balance cannot cover the doubled fee. */
|
|
42
|
+
const FEE_INSUFFICIENT = "DEPOSIT_FEE_INSUFFICIENT";
|
|
43
|
+
/**
|
|
44
|
+
* Work out what erasing these paths would destroy, without destroying anything.
|
|
45
|
+
*
|
|
46
|
+
* ⛔ ITS OWN STEP BECAUSE ONE CALLER HAS TO SHOW THE LIST BEFORE IT ASKS. `nmts erase` prints every
|
|
47
|
+
* file it is about to destroy and then waits for a typed sentence; folding this into the act
|
|
48
|
+
* would leave the terminal with nothing to print, and reading the list twice would leave the
|
|
49
|
+
* two reads free to disagree about what is in it.
|
|
50
|
+
*/
|
|
51
|
+
export async function planErase(input, paths) {
|
|
52
|
+
const list = await readFileList(input.server, input.apiKey, input.code, input.accountId);
|
|
53
|
+
const entries = list.manifest?.entries ?? [];
|
|
54
|
+
const index = buildIndex(entries);
|
|
55
|
+
const targets = resolving(() => batchTargets(entries, paths, { includeTrashed: true, nothingHappened: "Nothing was erased." }));
|
|
56
|
+
const files = uniqueById(targets.flatMap((t) => (t.kind === KIND_FILE ? [t] : filesUnder(entries, t.id))));
|
|
57
|
+
const going = uniqueById([...targets, ...files]);
|
|
58
|
+
if (files.length === 0) {
|
|
59
|
+
throw new DriveEditError("NOT_FOUND", `Nothing named holds a file; empty folders are removed with \`${BINARY_NAME} rm\`.`, {
|
|
60
|
+
exitCode: 4,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
files: files.map((f) => ({ id: f.id, path: fullPathOf(index, f) })),
|
|
65
|
+
going: going.map((e) => e.id),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Destroy what the plan names. ⛔ Irreversible, and nothing below asks whether the caller meant it.
|
|
70
|
+
*/
|
|
71
|
+
export async function eraseFiles(input, plan, options = {}) {
|
|
72
|
+
const releases = [];
|
|
73
|
+
if (options.releaseStorage === true) {
|
|
74
|
+
for (const f of plan.files) {
|
|
75
|
+
const path = f.path;
|
|
76
|
+
try {
|
|
77
|
+
const reply = await request(input.server, `/v1/items/${encodeURIComponent(f.id)}/release-storage`, {
|
|
78
|
+
method: "POST",
|
|
79
|
+
body: {},
|
|
80
|
+
token: input.apiKey,
|
|
81
|
+
accountProof: input.accountProof,
|
|
82
|
+
});
|
|
83
|
+
releases.push({ path, refused: null, ...counts(reply), ...fee(reply) });
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
// ⛔ THE ONE REFUSAL THAT STOPS THE RUN. A file with no deposit pays twice the chain fee
|
|
87
|
+
// out of the balance, and a balance that cannot cover it means the release did not
|
|
88
|
+
// happen — erasing behind it would destroy the account's key to bytes that are still
|
|
89
|
+
// being served and still being paid for. Nothing is erased, and the two numbers say
|
|
90
|
+
// exactly how far short the balance is.
|
|
91
|
+
if (error instanceof ServerError && error.code === FEE_INSUFFICIENT) {
|
|
92
|
+
throw new NmtsError(`${path}: releasing its storage costs ${amount(error, "needed_credits")} credits and this ` +
|
|
93
|
+
`account has ${amount(error, "balance_credits")}. It has no deposit, so the fee comes out of the balance.`, {
|
|
94
|
+
exitCode: 4,
|
|
95
|
+
nextStep: `Nothing was erased. Buy credits and run this again, or leave --release-storage off ` +
|
|
96
|
+
`to erase the file and let its storage run out on its own.`,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
// ⚠ "Not ours to destroy" is an answer, not a failure: the storage was bought by the
|
|
100
|
+
// wallet, and the erase goes on. A refusal of the KEY or the proof, and anything the
|
|
101
|
+
// server could not do, stops the run before a row is touched.
|
|
102
|
+
if (error instanceof ServerError && error.status !== 401 && error.status !== 403 && error.status < 500) {
|
|
103
|
+
releases.push({ path, released: 0, alreadyReleased: 0, failed: 0, feeCredits: 0, fromDeposit: false, refused: error.message });
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
throw error;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
let erased = 0;
|
|
111
|
+
for (let i = 0; i < plan.files.length; i += BATCH) {
|
|
112
|
+
const reply = await request(input.server, "/v1/items/erase", {
|
|
113
|
+
method: "POST",
|
|
114
|
+
body: { item_ids: plan.files.slice(i, i + BATCH).map((f) => f.id) },
|
|
115
|
+
token: input.apiKey,
|
|
116
|
+
accountProof: input.accountProof,
|
|
117
|
+
});
|
|
118
|
+
erased += typeof reply === "object" && reply !== null && typeof Reflect.get(reply, "erased") === "number"
|
|
119
|
+
? Number(Reflect.get(reply, "erased"))
|
|
120
|
+
: 0;
|
|
121
|
+
}
|
|
122
|
+
// ⛔ THE LIST GOES LAST, and re-decided against the list as it is on this attempt: only the
|
|
123
|
+
// ids this run erased leave it, whatever another device wrote in between.
|
|
124
|
+
const ids = new Set(plan.going);
|
|
125
|
+
const result = await applyToList(input, (current) => {
|
|
126
|
+
const still = current.filter((e) => ids.has(e.id)).map((e) => e.id);
|
|
127
|
+
return still.length === 0 ? null : { op: "purge", ids: still };
|
|
128
|
+
});
|
|
129
|
+
return { erased, files: [...plan.files], releases, seq: result.seq };
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Resolve these paths and destroy what they name, in one call.
|
|
133
|
+
*
|
|
134
|
+
* ⚠ FOR A CALLER THAT HAS ALREADY DECIDED. Anything that shows a person what is about to go should
|
|
135
|
+
* use `planErase` first, so what it shows is what it then destroys.
|
|
136
|
+
*/
|
|
137
|
+
export async function erasePaths(input, paths, options = {}) {
|
|
138
|
+
return eraseFiles(input, await planErase(input, paths), options);
|
|
139
|
+
}
|
|
140
|
+
/** The three counts a release answers with, read defensively. */
|
|
141
|
+
function counts(reply) {
|
|
142
|
+
const n = (name) => {
|
|
143
|
+
const v = typeof reply === "object" && reply !== null ? Reflect.get(reply, name) : undefined;
|
|
144
|
+
return typeof v === "number" ? v : 0;
|
|
145
|
+
};
|
|
146
|
+
return { released: n("released"), alreadyReleased: n("already_released"), failed: n("failed") };
|
|
147
|
+
}
|
|
148
|
+
/** What it cost, read the same defensive way. An older server says neither, which reads as 0. */
|
|
149
|
+
function fee(reply) {
|
|
150
|
+
const at = (name) => typeof reply === "object" && reply !== null ? Reflect.get(reply, name) : undefined;
|
|
151
|
+
const charged = at("fee_credits");
|
|
152
|
+
return {
|
|
153
|
+
feeCredits: typeof charged === "number" ? charged : 0,
|
|
154
|
+
fromDeposit: at("from_deposit") === true,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
/** One credit amount out of a refusal's details, or `?` when the server did not name it. */
|
|
158
|
+
function amount(error, field) {
|
|
159
|
+
const value = error.details[field];
|
|
160
|
+
return typeof value === "number" ? String(value) : "?";
|
|
161
|
+
}
|
package/dist/platform-sign.d.ts
CHANGED
|
@@ -25,14 +25,22 @@ export declare const DELEGATION_MAX_TTL_SECS = 2592000;
|
|
|
25
25
|
* ⛔ THE FIRST THREE ARE AN API KEY'S OWN SCOPES, value for value. One vocabulary, so that a
|
|
26
26
|
* business asking for "read and write" asks for the number a person's key already means by it.
|
|
27
27
|
* `register` is the fourth and opens exactly one door: making the account the token names.
|
|
28
|
+
*
|
|
29
|
+
* ⛔ `files_erase` IS THE FIFTH, AND AN API KEY HAS NO BIT OF THAT NAME. It opens the two acts
|
|
30
|
+
* nothing undoes: erasing a file's record and this account's key to it, and destroying the
|
|
31
|
+
* treasury's storage under a file. Its own name rather than `files_write`, so a token minted so
|
|
32
|
+
* an app can upload cannot destroy — and it does not stand alone: both doors also ask for the
|
|
33
|
+
* account code's own proof on the same request, which is an act only whoever holds the code can
|
|
34
|
+
* perform. The token is the business's permission; the proof is the key holder's.
|
|
28
35
|
*/
|
|
29
36
|
export declare const SCOPE_BITS: {
|
|
30
37
|
readonly files_read: 1;
|
|
31
38
|
readonly files_write: 2;
|
|
32
39
|
readonly storage_spend: 4;
|
|
33
40
|
readonly register: 8;
|
|
41
|
+
readonly files_erase: 16;
|
|
34
42
|
};
|
|
35
|
-
/** One of the
|
|
43
|
+
/** One of the five names above. */
|
|
36
44
|
export type ScopeName = keyof typeof SCOPE_BITS;
|
|
37
45
|
/** Every bit a token may carry. */
|
|
38
46
|
export declare const SCOPE_ALL: number;
|
package/dist/platform-sign.js
CHANGED
|
@@ -53,15 +53,23 @@ export const DELEGATION_MAX_TTL_SECS = 2_592_000;
|
|
|
53
53
|
* ⛔ THE FIRST THREE ARE AN API KEY'S OWN SCOPES, value for value. One vocabulary, so that a
|
|
54
54
|
* business asking for "read and write" asks for the number a person's key already means by it.
|
|
55
55
|
* `register` is the fourth and opens exactly one door: making the account the token names.
|
|
56
|
+
*
|
|
57
|
+
* ⛔ `files_erase` IS THE FIFTH, AND AN API KEY HAS NO BIT OF THAT NAME. It opens the two acts
|
|
58
|
+
* nothing undoes: erasing a file's record and this account's key to it, and destroying the
|
|
59
|
+
* treasury's storage under a file. Its own name rather than `files_write`, so a token minted so
|
|
60
|
+
* an app can upload cannot destroy — and it does not stand alone: both doors also ask for the
|
|
61
|
+
* account code's own proof on the same request, which is an act only whoever holds the code can
|
|
62
|
+
* perform. The token is the business's permission; the proof is the key holder's.
|
|
56
63
|
*/
|
|
57
64
|
export const SCOPE_BITS = {
|
|
58
65
|
files_read: 1,
|
|
59
66
|
files_write: 2,
|
|
60
67
|
storage_spend: 4,
|
|
61
68
|
register: 8,
|
|
69
|
+
files_erase: 16,
|
|
62
70
|
};
|
|
63
71
|
/** Every bit a token may carry. */
|
|
64
|
-
export const SCOPE_ALL = SCOPE_BITS.files_read | SCOPE_BITS.files_write | SCOPE_BITS.storage_spend | SCOPE_BITS.register;
|
|
72
|
+
export const SCOPE_ALL = SCOPE_BITS.files_read | SCOPE_BITS.files_write | SCOPE_BITS.storage_spend | SCOPE_BITS.register | SCOPE_BITS.files_erase;
|
|
65
73
|
/**
|
|
66
74
|
* A new key pair, from the runtime's own random source.
|
|
67
75
|
*
|
package/dist/product.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ export declare const BINARY_NAME = "nmts";
|
|
|
9
9
|
* beside it. Here rather than in `main.ts` because the MCP server has to say it too, and a
|
|
10
10
|
* command importing the entry point is a cycle waiting to bite.
|
|
11
11
|
*/
|
|
12
|
-
export declare const VERSION = "0.
|
|
12
|
+
export declare const VERSION = "0.39.0";
|
|
13
13
|
/** Where the product lives, for messages that need to send somebody somewhere real. */
|
|
14
14
|
export declare const HOME_URL = "https://nmts.me";
|
|
15
15
|
/** The source, so a person holding only the built program can find what it was built from. */
|
package/dist/product.js
CHANGED
|
@@ -18,7 +18,7 @@ export const BINARY_NAME = "nmts";
|
|
|
18
18
|
* beside it. Here rather than in `main.ts` because the MCP server has to say it too, and a
|
|
19
19
|
* command importing the entry point is a cycle waiting to bite.
|
|
20
20
|
*/
|
|
21
|
-
export const VERSION = "0.
|
|
21
|
+
export const VERSION = "0.39.0";
|
|
22
22
|
/** Where the product lives, for messages that need to send somebody somewhere real. */
|
|
23
23
|
export const HOME_URL = "https://nmts.me";
|
|
24
24
|
/** The source, so a person holding only the built program can find what it was built from. */
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import type { Readable } from "node:stream";
|
|
2
|
+
import type { PlaintextSink } from "../download-sink.ts";
|
|
3
|
+
import type { ManifestEntry } from "../shared/lib/drive/manifest-codec.ts";
|
|
4
|
+
import type { DriveObject } from "./listing.ts";
|
|
5
|
+
import type { GatewayCredential } from "./sigv4.ts";
|
|
6
|
+
export interface DriveSource {
|
|
7
|
+
/** The account's live file list. Called per request; the caller decides what to cache. */
|
|
8
|
+
entries(): Promise<readonly ManifestEntry[]>;
|
|
9
|
+
/**
|
|
10
|
+
* Fetch, decrypt and deliver one file into the sink.
|
|
11
|
+
*
|
|
12
|
+
* ⛔ INJECTED RATHER THAN IMPORTED so this server can be driven by a real S3 client in a test
|
|
13
|
+
* without an account, a network and somebody's credits. A gateway whose only test is an
|
|
14
|
+
* end-to-end one is a gateway whose refusals are never tested at all.
|
|
15
|
+
*/
|
|
16
|
+
fetch(object: DriveObject, sink: PlaintextSink): Promise<void>;
|
|
17
|
+
/**
|
|
18
|
+
* How to change the drive, when this machine has agreed to spending.
|
|
19
|
+
*
|
|
20
|
+
* ⛔ ABSENT MEANS READ ONLY, AND THAT IS A REFUSAL RATHER THAN A GAP. Uploading spends credits,
|
|
21
|
+
* which is one of the three things this tool asks a person about once per machine, and a
|
|
22
|
+
* gateway cannot ask: its stdin is not a terminal and the caller is a program. So the
|
|
23
|
+
* agreement has to exist beforehand, and where it does not, every write says so.
|
|
24
|
+
*/
|
|
25
|
+
readonly write?: DriveWriter;
|
|
26
|
+
}
|
|
27
|
+
export interface DriveWriter {
|
|
28
|
+
/** Store `body` at this key. `size` is the byte count the client declared. */
|
|
29
|
+
put(key: string, body: Readable, size: number): Promise<void>;
|
|
30
|
+
/** Send one file to the trash, where it stays recoverable for thirty days. */
|
|
31
|
+
trash(object: DriveObject): Promise<void>;
|
|
32
|
+
/**
|
|
33
|
+
* Staging for uploads that arrive in pieces. Absent means this gateway refuses them.
|
|
34
|
+
*
|
|
35
|
+
* ⚠ Separate from `put` because the pieces have to land somewhere before they are one file, and
|
|
36
|
+
* where that is belongs to whoever is running this rather than to the protocol.
|
|
37
|
+
*/
|
|
38
|
+
readonly multipart?: {
|
|
39
|
+
begin(key: string): Promise<string>;
|
|
40
|
+
part(uploadId: string, partNumber: number, body: Readable, size: number, expectedSha256: string | null): Promise<string>;
|
|
41
|
+
complete(uploadId: string): Promise<string>;
|
|
42
|
+
abort(uploadId: string): Promise<void>;
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
export interface GatewayOptions {
|
|
46
|
+
/**
|
|
47
|
+
* Every pair that may sign a request here, each optionally held to named buckets.
|
|
48
|
+
*
|
|
49
|
+
* ⛔ A LIST RATHER THAN ONE PAIR BECAUSE A BUCKET IS AN ACCOUNT. `nmts s3` makes one pair for one
|
|
50
|
+
* drive; a business serving many of its users' accounts hands each of them a pair of their
|
|
51
|
+
* own, and the restriction on the pair is what stops one customer reading another's bucket.
|
|
52
|
+
*/
|
|
53
|
+
readonly credentials: readonly GatewayCredential[];
|
|
54
|
+
/**
|
|
55
|
+
* Which drive answers to this bucket name, or null when none does.
|
|
56
|
+
*
|
|
57
|
+
* ⛔ THE GATEWAY DOES NOT KNOW WHAT A BUCKET IS. It was one name and one drive for as long as the
|
|
58
|
+
* only caller was the command-line tool; asked by a business's server it is a lookup that
|
|
59
|
+
* server does, and one it may do differently per name. What must not change is that a name
|
|
60
|
+
* this resolver refuses looks exactly like a name the caller may not touch (see below).
|
|
61
|
+
*/
|
|
62
|
+
readonly bucketOf: (name: string) => DriveSource | null | Promise<DriveSource | null>;
|
|
63
|
+
/**
|
|
64
|
+
* The names `ListBuckets` answers with, before the signing pair's own restriction is applied.
|
|
65
|
+
*
|
|
66
|
+
* ⚠ ABSENT IS A REAL ANSWER RATHER THAN A GAP. A gateway in front of a business's own lookup
|
|
67
|
+
* cannot enumerate its customers, so what it can honestly name is what the presented pair is
|
|
68
|
+
* held to — and an unrestricted pair on such a gateway is told nothing, which is true.
|
|
69
|
+
*/
|
|
70
|
+
readonly bucketNames?: () => readonly string[] | Promise<readonly string[]>;
|
|
71
|
+
/** Called with one line whenever a request is answered, so a person can watch what a tool does. */
|
|
72
|
+
readonly log?: (line: string) => void;
|
|
73
|
+
/** Passed in so a test can hold the clock still. */
|
|
74
|
+
readonly now?: () => number;
|
|
75
|
+
/**
|
|
76
|
+
* The sentence a write gets from a read-only drive. `nmts s3` says what a person runs on this
|
|
77
|
+
* machine to allow spending; a gateway somebody else runs has a different way in, and says its own.
|
|
78
|
+
*/
|
|
79
|
+
readonly readOnlyBecause?: string | undefined;
|
|
80
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { IncomingMessage, ServerResponse } from "node:http";
|
|
2
|
+
import type { GatewayOptions } from "./contract.ts";
|
|
3
|
+
export declare function fail(res: ServerResponse, status: number, code: string, message: string, resource: string): void;
|
|
4
|
+
export declare function handle(req: IncomingMessage, res: ServerResponse, options: GatewayOptions): Promise<void>;
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
// Which answer an S3 request gets: the signature first, then the bucket, then the verb.
|
|
2
|
+
//
|
|
3
|
+
// ⛔ WHAT IS NOT ANSWERED IS REFUSED, LOUDLY. An S3 client that asks for something this gateway does
|
|
4
|
+
// not do gets 501 and a sentence naming what it does do. The alternative -- answering an empty
|
|
5
|
+
// listing, or a 200 with nothing behind it -- is how a backup tool reports success over a backup
|
|
6
|
+
// that never happened.
|
|
7
|
+
import { listObjects, objectsOf, folderPrefixesOf, MAX_KEYS_LIMIT } from "./listing.js";
|
|
8
|
+
import { handleMultipart, isMultipartRequest } from "./multipart.js";
|
|
9
|
+
import { responseSink } from "./response-sink.js";
|
|
10
|
+
import { isKeyConflict } from "./same-file.js";
|
|
11
|
+
import { STREAMING_PAYLOAD, STREAMING_PAYLOAD_TRAILER, verifyAgainst, } from "./sigv4.js";
|
|
12
|
+
import { errorXml, listBucketsXml, listObjectsXml } from "./xml.js";
|
|
13
|
+
/**
|
|
14
|
+
* What a caller signed with a pair it may not use here.
|
|
15
|
+
*
|
|
16
|
+
* ⛔ THE SAME ANSWER WHETHER OR NOT THE BUCKET EXISTS, which is why the restriction is checked
|
|
17
|
+
* before the resolver is asked. Answering `NoSuchBucket` for a name the caller may not touch
|
|
18
|
+
* would turn this gateway into a way of asking "does this business have a customer called…",
|
|
19
|
+
* one guess at a time.
|
|
20
|
+
*/
|
|
21
|
+
const NOT_YOURS = "That access key may not use that bucket.";
|
|
22
|
+
export function fail(res, status, code, message, resource) {
|
|
23
|
+
const body = errorXml(code, message, resource);
|
|
24
|
+
res.writeHead(status, { "content-type": "application/xml", "content-length": String(Buffer.byteLength(body)) });
|
|
25
|
+
res.end(body);
|
|
26
|
+
}
|
|
27
|
+
/** `/drive/photos/a.jpg` → bucket `drive`, key `photos/a.jpg`. */
|
|
28
|
+
function splitPath(pathname) {
|
|
29
|
+
const trimmed = pathname.replace(/^\//, "");
|
|
30
|
+
const at = trimmed.indexOf("/");
|
|
31
|
+
if (at < 0)
|
|
32
|
+
return { bucket: decodeURIComponent(trimmed), key: "" };
|
|
33
|
+
return { bucket: decodeURIComponent(trimmed.slice(0, at)), key: decodeURIComponent(trimmed.slice(at + 1)) };
|
|
34
|
+
}
|
|
35
|
+
function headerOf(req, name) {
|
|
36
|
+
const raw = req.headers[name];
|
|
37
|
+
return Array.isArray(raw) ? raw.join(",") : raw;
|
|
38
|
+
}
|
|
39
|
+
/** The one sentence a write gets from `nmts s3` when this machine has not agreed to spending. */
|
|
40
|
+
function readOnlyOnThisMachine() {
|
|
41
|
+
return ("This gateway is read only. Uploading spends credits, and this machine has not agreed to " +
|
|
42
|
+
"spending — `nmts consent grant spend`, run by the person whose account this is, is what " +
|
|
43
|
+
"changes that. Nothing was written.");
|
|
44
|
+
}
|
|
45
|
+
/** Whether the pair that signed is allowed anywhere near this bucket name. */
|
|
46
|
+
function mayTouch(credential, bucket) {
|
|
47
|
+
const only = credential.buckets;
|
|
48
|
+
return only === undefined || only.includes(bucket);
|
|
49
|
+
}
|
|
50
|
+
/** What `ListBuckets` says: what the gateway can name, narrowed to what this pair may touch. */
|
|
51
|
+
async function bucketsFor(options, credential) {
|
|
52
|
+
const only = credential.buckets;
|
|
53
|
+
if (options.bucketNames === undefined)
|
|
54
|
+
return only ?? [];
|
|
55
|
+
const named = await options.bucketNames();
|
|
56
|
+
return only === undefined ? named : named.filter((name) => only.includes(name));
|
|
57
|
+
}
|
|
58
|
+
function objectHeaders(object) {
|
|
59
|
+
return {
|
|
60
|
+
"content-type": "application/octet-stream",
|
|
61
|
+
"last-modified": new Date(object.entry.updatedAt).toUTCString(),
|
|
62
|
+
etag: object.etag,
|
|
63
|
+
"accept-ranges": "none",
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
export async function handle(req, res, options) {
|
|
67
|
+
const url = req.url ?? "/";
|
|
68
|
+
const at = url.indexOf("?");
|
|
69
|
+
const pathname = at < 0 ? url : url.slice(0, at);
|
|
70
|
+
const query = new URLSearchParams(at < 0 ? "" : url.slice(at + 1));
|
|
71
|
+
const method = (req.method ?? "GET").toUpperCase();
|
|
72
|
+
const verdict = verifyAgainst({ method, url, headers: req.headers }, options.credentials, options.now?.() ?? Date.now());
|
|
73
|
+
if (!verdict.ok) {
|
|
74
|
+
fail(res, 403, verdict.code, verdict.message, pathname);
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
const credential = verdict.credential;
|
|
78
|
+
const { bucket, key } = splitPath(pathname);
|
|
79
|
+
if (pathname === "/" && (method === "GET" || method === "HEAD")) {
|
|
80
|
+
const body = listBucketsXml(await bucketsFor(options, credential), new Date(0).toISOString());
|
|
81
|
+
res.writeHead(200, { "content-type": "application/xml", "content-length": String(Buffer.byteLength(body)) });
|
|
82
|
+
res.end(method === "HEAD" ? undefined : body);
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
if (!mayTouch(credential, bucket)) {
|
|
86
|
+
fail(res, 403, "AccessDenied", NOT_YOURS, pathname);
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
const source = await options.bucketOf(bucket);
|
|
90
|
+
if (source === null) {
|
|
91
|
+
fail(res, 404, "NoSuchBucket", `No bucket named ${bucket} is served here.`, pathname);
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
// ⛔ MEASURED, NOT GUESSED: rclone's first act when copying a file is to create the bucket, and a
|
|
95
|
+
// refusal here ends the copy before the upload is ever attempted. The bucket exists, so the
|
|
96
|
+
// honest answer to "make it" is that it is made.
|
|
97
|
+
if (key === "" && method === "PUT") {
|
|
98
|
+
res.writeHead(200, { "content-length": "0" });
|
|
99
|
+
res.end();
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
const entries = await source.entries();
|
|
103
|
+
if (key === "" && (method === "GET" || method === "HEAD")) {
|
|
104
|
+
const objects = objectsOf(entries);
|
|
105
|
+
const listing = listObjects(objects, folderPrefixesOf(entries), {
|
|
106
|
+
prefix: query.get("prefix") ?? "",
|
|
107
|
+
delimiter: query.get("delimiter") ?? "",
|
|
108
|
+
maxKeys: Number(query.get("max-keys") ?? MAX_KEYS_LIMIT) || MAX_KEYS_LIMIT,
|
|
109
|
+
after: query.get("continuation-token") ?? query.get("start-after") ?? query.get("marker"),
|
|
110
|
+
});
|
|
111
|
+
const body = listObjectsXml({
|
|
112
|
+
bucket,
|
|
113
|
+
prefix: query.get("prefix") ?? "",
|
|
114
|
+
delimiter: query.get("delimiter") ?? "",
|
|
115
|
+
maxKeys: Number(query.get("max-keys") ?? MAX_KEYS_LIMIT) || MAX_KEYS_LIMIT,
|
|
116
|
+
v2: query.get("list-type") === "2",
|
|
117
|
+
contents: listing.contents,
|
|
118
|
+
commonPrefixes: listing.commonPrefixes,
|
|
119
|
+
truncated: listing.truncated,
|
|
120
|
+
next: listing.next,
|
|
121
|
+
encodingType: query.get("encoding-type"),
|
|
122
|
+
});
|
|
123
|
+
res.writeHead(200, { "content-type": "application/xml", "content-length": String(Buffer.byteLength(body)) });
|
|
124
|
+
res.end(method === "HEAD" ? undefined : body);
|
|
125
|
+
options.log?.(`${method} list prefix=${query.get("prefix") ?? ""} → ${listing.contents.length} keys`);
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
if (method === "HEAD" || method === "GET") {
|
|
129
|
+
const object = objectsOf(entries).find((o) => o.key === key);
|
|
130
|
+
if (object === undefined) {
|
|
131
|
+
fail(res, 404, "NoSuchKey", "This account's file list has no such file.", pathname);
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
if (method === "HEAD") {
|
|
135
|
+
res.writeHead(200, { ...objectHeaders(object), "content-length": String(object.size) });
|
|
136
|
+
res.end();
|
|
137
|
+
options.log?.(`HEAD ${key}`);
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
if (object.entry.dekWrapped === undefined) {
|
|
141
|
+
fail(res, 500, "InternalError", "That entry has no key in the file list.", pathname);
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
const sink = responseSink(res, { headers: objectHeaders(object) });
|
|
145
|
+
try {
|
|
146
|
+
await source.fetch(object, sink);
|
|
147
|
+
options.log?.(`GET ${key} → ${object.size} bytes`);
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
await sink.abandon();
|
|
151
|
+
if (!res.headersSent) {
|
|
152
|
+
fail(res, 502, "InternalError", error instanceof Error ? error.message : String(error), pathname);
|
|
153
|
+
}
|
|
154
|
+
options.log?.(`GET ${key} → failed`);
|
|
155
|
+
}
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
const writer = source.write;
|
|
159
|
+
// ⛔ WHETHER A TAKEN KEY IS A CONFLICT IS NOT DECIDED HERE.
|
|
160
|
+
// It used to be, on the strength of the NAME alone, and both upload paths carried their own
|
|
161
|
+
// copy of that check. The question is now about CONTENT — is the file arriving the file
|
|
162
|
+
// already there — and it cannot be answered until the bytes have arrived, so it is answered
|
|
163
|
+
// once, by the writer, at the point both paths meet. What reaches this layer is the verdict:
|
|
164
|
+
// a writer that returns normally means the key now holds these bytes (whether it had to send
|
|
165
|
+
// them or they were already there), and one that throws a conflict means something else is at
|
|
166
|
+
// that key. ⭐ The status matters: 409 is a request the drive declined, 500 is a fault of ours.
|
|
167
|
+
const refuseConflict = (error) => {
|
|
168
|
+
fail(res, 409, "InvalidRequest", error instanceof Error ? error.message : String(error), pathname);
|
|
169
|
+
};
|
|
170
|
+
if (isMultipartRequest(method, query) && key !== "") {
|
|
171
|
+
if (writer === undefined) {
|
|
172
|
+
fail(res, 501, "NotImplemented", options.readOnlyBecause ?? readOnlyOnThisMachine(), pathname);
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
const handled = await handleMultipart({
|
|
176
|
+
req,
|
|
177
|
+
res,
|
|
178
|
+
bucket,
|
|
179
|
+
key,
|
|
180
|
+
method,
|
|
181
|
+
query,
|
|
182
|
+
writer,
|
|
183
|
+
payloadHash: /^[0-9a-f]{64}$/.test(verdict.payloadHash) ? verdict.payloadHash : null,
|
|
184
|
+
fail: (status, code, message) => fail(res, status, code, message, pathname),
|
|
185
|
+
...(options.log === undefined ? {} : { log: options.log }),
|
|
186
|
+
});
|
|
187
|
+
if (handled)
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
if (method === "PUT" && key !== "") {
|
|
191
|
+
if (writer === undefined) {
|
|
192
|
+
fail(res, 501, "NotImplemented", options.readOnlyBecause ?? readOnlyOnThisMachine(), pathname);
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
const declared = headerOf(req, "x-amz-content-sha256");
|
|
196
|
+
if (declared === STREAMING_PAYLOAD || declared === STREAMING_PAYLOAD_TRAILER) {
|
|
197
|
+
fail(res, 501, "NotImplemented", "This gateway does not read chunk-signed uploads yet. Tell the client to send the body " +
|
|
198
|
+
"unsigned (the AWS CLI calls this --no-sign-payload on http endpoints; rclone already " +
|
|
199
|
+
"does it).", pathname);
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
const length = Number(headerOf(req, "content-length") ?? "");
|
|
203
|
+
if (!Number.isInteger(length) || length < 0) {
|
|
204
|
+
fail(res, 411, "MissingContentLength", "This gateway needs to know the size before it starts.", pathname);
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
try {
|
|
208
|
+
await writer.put(key, req, length);
|
|
209
|
+
}
|
|
210
|
+
catch (error) {
|
|
211
|
+
if (isKeyConflict(error)) {
|
|
212
|
+
refuseConflict(error);
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
fail(res, 500, "InternalError", error instanceof Error ? error.message : String(error), pathname);
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
res.writeHead(200, { "content-length": "0" });
|
|
219
|
+
res.end();
|
|
220
|
+
options.log?.(`PUT ${key} → ${length} bytes`);
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
if (method === "DELETE" && key !== "") {
|
|
224
|
+
if (writer === undefined) {
|
|
225
|
+
fail(res, 501, "NotImplemented", options.readOnlyBecause ?? readOnlyOnThisMachine(), pathname);
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
const object = objectsOf(entries).find((o) => o.key === key);
|
|
229
|
+
if (object === undefined) {
|
|
230
|
+
// S3 answers 204 for a key that is not there, and clients rely on it: a sync that deletes
|
|
231
|
+
// the same key twice must not fail the second time.
|
|
232
|
+
res.writeHead(204);
|
|
233
|
+
res.end();
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
try {
|
|
237
|
+
await writer.trash(object);
|
|
238
|
+
}
|
|
239
|
+
catch (error) {
|
|
240
|
+
fail(res, 500, "InternalError", error instanceof Error ? error.message : String(error), pathname);
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
res.writeHead(204);
|
|
244
|
+
res.end();
|
|
245
|
+
options.log?.(`DELETE ${key} → trash`);
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
fail(res, 501, "NotImplemented", `This gateway does not answer ${method} on that address.`, pathname);
|
|
249
|
+
}
|