@poe-platform/safe-fs 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +290 -0
- package/dist/safe-fs/bridge/confinement.d.ts +3 -0
- package/dist/safe-fs/bridge/confinement.js +171 -0
- package/dist/safe-fs/bridge/filesystem.d.ts +88 -0
- package/dist/safe-fs/bridge/filesystem.js +411 -0
- package/dist/safe-fs/bridge/index.d.ts +6 -0
- package/dist/safe-fs/bridge/index.js +18 -0
- package/dist/safe-fs/bridge/stats.d.ts +4 -0
- package/dist/safe-fs/bridge/stats.js +37 -0
- package/dist/safe-fs/bridge/types.d.ts +75 -0
- package/dist/safe-fs/bridge/types.js +1 -0
- package/dist/safe-fs/bridge/values.d.ts +6 -0
- package/dist/safe-fs/bridge/values.js +65 -0
- package/dist/safe-fs/config/memory.d.ts +3 -0
- package/dist/safe-fs/config/memory.js +9 -0
- package/dist/safe-fs/config/real.d.ts +5 -0
- package/dist/safe-fs/config/real.js +16 -0
- package/dist/safe-fs/config.d.ts +15 -0
- package/dist/safe-fs/config.js +46 -0
- package/dist/safe-fs/config.node.d.ts +2 -0
- package/dist/safe-fs/config.node.js +22 -0
- package/dist/safe-fs/contracts/abort.d.ts +5 -0
- package/dist/safe-fs/contracts/abort.js +34 -0
- package/dist/safe-fs/contracts/cleanup.d.ts +1 -0
- package/dist/safe-fs/contracts/cleanup.js +9 -0
- package/dist/safe-fs/contracts/errors.d.ts +50 -0
- package/dist/safe-fs/contracts/errors.js +77 -0
- package/dist/safe-fs/contracts/filesystem.d.ts +96 -0
- package/dist/safe-fs/contracts/filesystem.js +1 -0
- package/dist/safe-fs/contracts/index.d.ts +4 -0
- package/dist/safe-fs/contracts/index.js +4 -0
- package/dist/safe-fs/contracts/io.d.ts +8 -0
- package/dist/safe-fs/contracts/io.js +92 -0
- package/dist/safe-fs/contracts/path.d.ts +8 -0
- package/dist/safe-fs/contracts/path.js +8 -0
- package/dist/safe-fs/contracts/portable-path.d.ts +13 -0
- package/dist/safe-fs/contracts/portable-path.js +57 -0
- package/dist/safe-fs/contracts/virtual-path.d.ts +7 -0
- package/dist/safe-fs/contracts/virtual-path.js +66 -0
- package/dist/safe-fs/core.d.ts +12 -0
- package/dist/safe-fs/core.js +12 -0
- package/dist/safe-fs/fs/memory/index.d.ts +58 -0
- package/dist/safe-fs/fs/memory/index.js +532 -0
- package/dist/safe-fs/fs/mount/comparison.d.ts +19 -0
- package/dist/safe-fs/fs/mount/comparison.js +118 -0
- package/dist/safe-fs/fs/mount/identity.d.ts +2 -0
- package/dist/safe-fs/fs/mount/identity.js +12 -0
- package/dist/safe-fs/fs/mount/index.d.ts +44 -0
- package/dist/safe-fs/fs/mount/index.js +546 -0
- package/dist/safe-fs/fs/overlay/index.d.ts +73 -0
- package/dist/safe-fs/fs/overlay/index.js +927 -0
- package/dist/safe-fs/fs/readonly/index.d.ts +31 -0
- package/dist/safe-fs/fs/readonly/index.js +128 -0
- package/dist/safe-fs/fs/real/allocation.d.ts +1 -0
- package/dist/safe-fs/fs/real/allocation.js +8 -0
- package/dist/safe-fs/fs/real/index.d.ts +76 -0
- package/dist/safe-fs/fs/real/index.js +543 -0
- package/dist/safe-fs/fs/s3/authority.d.ts +7 -0
- package/dist/safe-fs/fs/s3/authority.js +31 -0
- package/dist/safe-fs/fs/s3/filesystem.d.ts +93 -0
- package/dist/safe-fs/fs/s3/filesystem.js +1024 -0
- package/dist/safe-fs/fs/s3/http/index.d.ts +2 -0
- package/dist/safe-fs/fs/s3/http/index.js +1 -0
- package/dist/safe-fs/fs/s3/http/request.d.ts +16 -0
- package/dist/safe-fs/fs/s3/http/request.js +196 -0
- package/dist/safe-fs/fs/s3/http/signature.d.ts +20 -0
- package/dist/safe-fs/fs/s3/http/signature.js +74 -0
- package/dist/safe-fs/fs/s3/http/transport.d.ts +3 -0
- package/dist/safe-fs/fs/s3/http/transport.js +455 -0
- package/dist/safe-fs/fs/s3/http/types.d.ts +30 -0
- package/dist/safe-fs/fs/s3/http/types.js +1 -0
- package/dist/safe-fs/fs/s3/http/xml.d.ts +11 -0
- package/dist/safe-fs/fs/s3/http/xml.js +188 -0
- package/dist/safe-fs/fs/s3/index.d.ts +6 -0
- package/dist/safe-fs/fs/s3/index.js +3 -0
- package/dist/safe-fs/fs/s3/mock.d.ts +48 -0
- package/dist/safe-fs/fs/s3/mock.js +233 -0
- package/dist/safe-fs/fs/s3/registry.d.ts +10 -0
- package/dist/safe-fs/fs/s3/registry.js +71 -0
- package/dist/safe-fs/fs/s3/transport.d.ts +103 -0
- package/dist/safe-fs/fs/s3/transport.js +28 -0
- package/dist/safe-fs/fs/webdav/index.d.ts +2 -0
- package/dist/safe-fs/fs/webdav/index.js +1 -0
- package/dist/safe-fs/fs/webdav/resource-id.d.ts +18 -0
- package/dist/safe-fs/fs/webdav/resource-id.js +109 -0
- package/dist/safe-fs/fs/webdav/webdav.d.ts +101 -0
- package/dist/safe-fs/fs/webdav/webdav.js +1226 -0
- package/dist/safe-fs/fs/webdav/xml.d.ts +15 -0
- package/dist/safe-fs/fs/webdav/xml.js +250 -0
- package/dist/safe-fs/index.d.ts +14 -0
- package/dist/safe-fs/index.js +12 -0
- package/dist/safe-fs/node/filesystem.d.ts +11 -0
- package/dist/safe-fs/node/filesystem.js +28 -0
- package/dist/safe-fs/node/index.d.ts +2 -0
- package/dist/safe-fs/node/index.js +1 -0
- package/dist/safe-fs/node/stats.d.ts +1 -0
- package/dist/safe-fs/node/stats.js +1 -0
- package/dist/safe-fs/node/values.d.ts +1 -0
- package/dist/safe-fs/node/values.js +1 -0
- package/dist/safe-fs/node-host.d.ts +1 -0
- package/dist/safe-fs/node-host.js +1 -0
- package/dist/safe-fs/node-unavailable.d.ts +1 -0
- package/dist/safe-fs/node-unavailable.js +1 -0
- package/dist/safe-fs/platform/browser.d.ts +12 -0
- package/dist/safe-fs/platform/browser.js +39 -0
- package/dist/safe-fs/platform/node.d.ts +13 -0
- package/dist/safe-fs/platform/node.js +26 -0
- package/package.json +108 -0
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import { S3ServiceError } from "../transport.js";
|
|
2
|
+
export function malformed(message = "invalid S3 XML response") {
|
|
3
|
+
throw new S3ServiceError("InvalidResponse", 502, message);
|
|
4
|
+
}
|
|
5
|
+
function validCharacter(code) {
|
|
6
|
+
return code === 9 || code === 10 || code === 13 || (code >= 32 && code <= 0xd7ff)
|
|
7
|
+
|| (code >= 0xe000 && code <= 0xfffd) || (code >= 0x10000 && code <= 0x10ffff);
|
|
8
|
+
}
|
|
9
|
+
function decodeEntities(text) {
|
|
10
|
+
let result = "";
|
|
11
|
+
let position = 0;
|
|
12
|
+
while (position < text.length) {
|
|
13
|
+
const start = text.indexOf("&", position);
|
|
14
|
+
if (start < 0)
|
|
15
|
+
return result + text.slice(position);
|
|
16
|
+
result += text.slice(position, start);
|
|
17
|
+
const end = text.indexOf(";", start + 1);
|
|
18
|
+
if (end < 0 || end - start > 12)
|
|
19
|
+
malformed("invalid XML entity");
|
|
20
|
+
const entity = text.slice(start + 1, end);
|
|
21
|
+
const named = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" };
|
|
22
|
+
if (Object.hasOwn(named, entity))
|
|
23
|
+
result += named[entity];
|
|
24
|
+
else {
|
|
25
|
+
const hexadecimal = entity.startsWith("#x");
|
|
26
|
+
const digits = entity.slice(hexadecimal ? 2 : 1);
|
|
27
|
+
if (!entity.startsWith("#") || !(hexadecimal ? /^[0-9a-fA-F]+$/ : /^[0-9]+$/).test(digits))
|
|
28
|
+
malformed("unknown XML entity");
|
|
29
|
+
const code = Number.parseInt(digits, hexadecimal ? 16 : 10);
|
|
30
|
+
if (!validCharacter(code))
|
|
31
|
+
malformed("invalid XML code point");
|
|
32
|
+
result += String.fromCodePoint(code);
|
|
33
|
+
}
|
|
34
|
+
position = end + 1;
|
|
35
|
+
}
|
|
36
|
+
return result;
|
|
37
|
+
}
|
|
38
|
+
export function parseXml(bytes) {
|
|
39
|
+
let xml;
|
|
40
|
+
try {
|
|
41
|
+
xml = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
return malformed("XML is not UTF-8");
|
|
45
|
+
}
|
|
46
|
+
for (const character of xml)
|
|
47
|
+
if (!validCharacter(character.codePointAt(0)))
|
|
48
|
+
malformed("invalid XML character");
|
|
49
|
+
xml = xml.replace(/\r\n?/g, "\n");
|
|
50
|
+
let position = 0;
|
|
51
|
+
let nodes = 0;
|
|
52
|
+
let root;
|
|
53
|
+
const stack = [];
|
|
54
|
+
const whitespace = () => { while (position < xml.length && /[ \t\r\n]/.test(xml[position]))
|
|
55
|
+
position++; };
|
|
56
|
+
const name = () => {
|
|
57
|
+
const start = position;
|
|
58
|
+
if (!/[A-Za-z_]/.test(xml[position] ?? ""))
|
|
59
|
+
malformed("invalid XML name");
|
|
60
|
+
while (position < xml.length && /[A-Za-z0-9_.:-]/.test(xml[position]))
|
|
61
|
+
position++;
|
|
62
|
+
const result = xml.slice(start, position);
|
|
63
|
+
if (result.length > 256)
|
|
64
|
+
malformed("XML name too long");
|
|
65
|
+
return result;
|
|
66
|
+
};
|
|
67
|
+
while (position < xml.length) {
|
|
68
|
+
if (xml[position] !== "<") {
|
|
69
|
+
const end = xml.indexOf("<", position);
|
|
70
|
+
const text = xml.slice(position, end < 0 ? xml.length : end);
|
|
71
|
+
if (text.includes("]]>"))
|
|
72
|
+
malformed();
|
|
73
|
+
const current = stack.at(-1);
|
|
74
|
+
if (current)
|
|
75
|
+
current.text += decodeEntities(text);
|
|
76
|
+
else if (text.trim())
|
|
77
|
+
malformed("text outside XML root");
|
|
78
|
+
position = end < 0 ? xml.length : end;
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
if (xml.startsWith("<?xml ", position) && !root && position === 0) {
|
|
82
|
+
const end = xml.indexOf("?>", position);
|
|
83
|
+
if (end < 0 || end > 256)
|
|
84
|
+
malformed();
|
|
85
|
+
const declaration = xml.slice(position, end);
|
|
86
|
+
if (!/^<\?xml\s+version\s*=\s*(?:"1\.0"|'1\.0')(?:\s+encoding\s*=\s*(?:"[Uu][Tt][Ff]-8"|'[Uu][Tt][Ff]-8'))?(?:\s+standalone\s*=\s*(?:"(?:yes|no)"|'(?:yes|no)'))?\s*$/.test(declaration))
|
|
87
|
+
malformed("unsupported XML declaration");
|
|
88
|
+
position = end + 2;
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
if (xml.startsWith("<!--", position)) {
|
|
92
|
+
const end = xml.indexOf("-->", position + 4);
|
|
93
|
+
const comment = xml.slice(position + 4, end);
|
|
94
|
+
if (end < 0 || comment.includes("--") || comment.endsWith("-"))
|
|
95
|
+
malformed();
|
|
96
|
+
position = end + 3;
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
if (xml.startsWith("<![CDATA[", position)) {
|
|
100
|
+
const end = xml.indexOf("]]>", position + 9);
|
|
101
|
+
const current = stack.at(-1);
|
|
102
|
+
if (end < 0 || !current)
|
|
103
|
+
malformed();
|
|
104
|
+
current.text += xml.slice(position + 9, end);
|
|
105
|
+
position = end + 3;
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
position++;
|
|
109
|
+
const closing = xml[position] === "/";
|
|
110
|
+
if (closing)
|
|
111
|
+
position++;
|
|
112
|
+
const tag = name();
|
|
113
|
+
if (closing) {
|
|
114
|
+
whitespace();
|
|
115
|
+
if (xml[position++] !== ">" || stack.pop()?.name !== tag)
|
|
116
|
+
malformed("mismatched XML close tag");
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
const attributes = new Set();
|
|
120
|
+
while (true) {
|
|
121
|
+
const previous = position;
|
|
122
|
+
whitespace();
|
|
123
|
+
if (xml[position] === ">" || xml[position] === "/")
|
|
124
|
+
break;
|
|
125
|
+
if (previous === position || attributes.size >= 32)
|
|
126
|
+
malformed("invalid XML attributes");
|
|
127
|
+
const attribute = name();
|
|
128
|
+
if (attributes.has(attribute))
|
|
129
|
+
malformed("duplicate XML attribute");
|
|
130
|
+
attributes.add(attribute);
|
|
131
|
+
whitespace();
|
|
132
|
+
if (xml[position++] !== "=")
|
|
133
|
+
malformed();
|
|
134
|
+
whitespace();
|
|
135
|
+
const quote = xml[position++];
|
|
136
|
+
if (quote !== '"' && quote !== "'")
|
|
137
|
+
malformed();
|
|
138
|
+
const end = xml.indexOf(quote, position);
|
|
139
|
+
if (end < 0 || xml.slice(position, end).includes("<"))
|
|
140
|
+
malformed();
|
|
141
|
+
decodeEntities(xml.slice(position, end));
|
|
142
|
+
position = end + 1;
|
|
143
|
+
}
|
|
144
|
+
const selfClosing = xml[position] === "/";
|
|
145
|
+
if (selfClosing)
|
|
146
|
+
position++;
|
|
147
|
+
if (xml[position++] !== ">")
|
|
148
|
+
malformed();
|
|
149
|
+
if (++nodes > 32_768 || stack.length >= 32)
|
|
150
|
+
malformed("XML structure limit exceeded");
|
|
151
|
+
const node = { name: tag, text: "", children: [] };
|
|
152
|
+
const parent = stack.at(-1);
|
|
153
|
+
if (parent)
|
|
154
|
+
parent.children.push(node);
|
|
155
|
+
else if (root)
|
|
156
|
+
malformed("multiple XML roots");
|
|
157
|
+
else
|
|
158
|
+
root = node;
|
|
159
|
+
if (!selfClosing)
|
|
160
|
+
stack.push(node);
|
|
161
|
+
}
|
|
162
|
+
if (!root || stack.length)
|
|
163
|
+
malformed("incomplete XML");
|
|
164
|
+
return root;
|
|
165
|
+
}
|
|
166
|
+
export function children(node, name) {
|
|
167
|
+
return node.children.filter(child => child.name === name);
|
|
168
|
+
}
|
|
169
|
+
export function text(node, name, required = false) {
|
|
170
|
+
const matches = children(node, name);
|
|
171
|
+
if (matches.length > 1 || (required && matches.length !== 1))
|
|
172
|
+
malformed(`invalid ${name} count`);
|
|
173
|
+
const child = matches[0];
|
|
174
|
+
if (child?.children.length)
|
|
175
|
+
malformed(`invalid ${name} value`);
|
|
176
|
+
return child?.text;
|
|
177
|
+
}
|
|
178
|
+
export function integer(value) {
|
|
179
|
+
if (!/^[0-9]+$/.test(value) || !Number.isSafeInteger(Number(value)))
|
|
180
|
+
malformed("invalid integer");
|
|
181
|
+
return Number(value);
|
|
182
|
+
}
|
|
183
|
+
export function timestamp(value) {
|
|
184
|
+
const date = new Date(value);
|
|
185
|
+
if (!Number.isFinite(date.getTime()))
|
|
186
|
+
malformed("invalid timestamp");
|
|
187
|
+
return date;
|
|
188
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { S3FileSystem, S3RenameError } from "./filesystem.js";
|
|
2
|
+
export type { S3FileSystemOptions } from "./filesystem.js";
|
|
3
|
+
export { MockS3Client } from "./mock.js";
|
|
4
|
+
export type { MockS3ClientOptions, MockS3Operation, MockS3Request } from "./mock.js";
|
|
5
|
+
export { createS3Transport, encodeCopySource, S3ServiceError } from "./transport.js";
|
|
6
|
+
export type { S3Body, S3Client, S3CopyInput, S3CopyOutput, S3DeleteInput, S3GetOutput, S3HeadOutput, S3ListInput, S3ListOutput, S3ObjectInput, S3ObjectSummary, S3PutInput, S3RequestOptions, S3Transport, S3TransportCapabilities, } from "./transport.js";
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { S3StreamGetInput, S3StreamGetOutput, S3StreamPutInput } from "./transport.js";
|
|
2
|
+
import type { S3CopyInput, S3CopyOutput, S3DeleteInput, S3GetOutput, S3HeadOutput, S3ListInput, S3ListOutput, S3ObjectInput, S3PutInput, S3RequestOptions, S3Transport } from "./transport.js";
|
|
3
|
+
export type MockS3Operation = "headObject" | "getObject" | "putObject" | "deleteObject" | "copyObject" | "listObjectsV2";
|
|
4
|
+
export interface MockS3Request {
|
|
5
|
+
readonly operation: MockS3Operation;
|
|
6
|
+
readonly input: S3ObjectInput | S3ListInput | S3CopyInput | S3PutInput | S3DeleteInput;
|
|
7
|
+
}
|
|
8
|
+
export interface MockS3ClientOptions {
|
|
9
|
+
readonly buckets: readonly string[];
|
|
10
|
+
readonly pageSize?: number;
|
|
11
|
+
readonly now?: () => Date;
|
|
12
|
+
readonly authorize?: (request: MockS3Request) => void | Promise<void>;
|
|
13
|
+
}
|
|
14
|
+
export declare class MockS3Client implements S3Transport {
|
|
15
|
+
readonly capabilities: Readonly<{
|
|
16
|
+
conditionalPut: true;
|
|
17
|
+
conditionalCopy: true;
|
|
18
|
+
conditionalDelete: true;
|
|
19
|
+
streamingRead: true;
|
|
20
|
+
streamingWrite: true;
|
|
21
|
+
}>;
|
|
22
|
+
private readonly buckets;
|
|
23
|
+
private readonly cursors;
|
|
24
|
+
private readonly history;
|
|
25
|
+
private readonly pageSize;
|
|
26
|
+
private readonly now;
|
|
27
|
+
private readonly authorize;
|
|
28
|
+
private nextToken;
|
|
29
|
+
constructor(options: MockS3ClientOptions);
|
|
30
|
+
get requests(): readonly MockS3Request[];
|
|
31
|
+
private begin;
|
|
32
|
+
private bucket;
|
|
33
|
+
private object;
|
|
34
|
+
private head;
|
|
35
|
+
private store;
|
|
36
|
+
headObject(input: S3ObjectInput, options?: S3RequestOptions): Promise<S3HeadOutput>;
|
|
37
|
+
getObject(input: S3ObjectInput, options?: S3RequestOptions): Promise<S3GetOutput>;
|
|
38
|
+
putObject(input: S3PutInput, options?: S3RequestOptions): Promise<{
|
|
39
|
+
ETag: string;
|
|
40
|
+
}>;
|
|
41
|
+
getObjectStream(input: S3StreamGetInput, options?: S3RequestOptions): Promise<S3StreamGetOutput>;
|
|
42
|
+
putObjectStream(input: S3StreamPutInput, options?: S3RequestOptions): Promise<{
|
|
43
|
+
ETag: string;
|
|
44
|
+
}>;
|
|
45
|
+
deleteObject(input: S3DeleteInput, options?: S3RequestOptions): Promise<Record<string, never>>;
|
|
46
|
+
copyObject(input: S3CopyInput, options?: S3RequestOptions): Promise<S3CopyOutput>;
|
|
47
|
+
listObjectsV2(input: S3ListInput, options?: S3RequestOptions): Promise<S3ListOutput>;
|
|
48
|
+
}
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { collectBytes } from "../../contracts/io.js";
|
|
3
|
+
import { recordMockS3Head } from "./authority.js";
|
|
4
|
+
import { S3ServiceError } from "./transport.js";
|
|
5
|
+
function compareKeys(left, right) {
|
|
6
|
+
return Buffer.compare(Buffer.from(left), Buffer.from(right));
|
|
7
|
+
}
|
|
8
|
+
export class MockS3Client {
|
|
9
|
+
capabilities = Object.freeze({ conditionalPut: true, conditionalCopy: true, conditionalDelete: true, streamingRead: true, streamingWrite: true });
|
|
10
|
+
buckets = new Map();
|
|
11
|
+
cursors = new Map();
|
|
12
|
+
history = [];
|
|
13
|
+
pageSize;
|
|
14
|
+
now;
|
|
15
|
+
authorize;
|
|
16
|
+
nextToken = 0;
|
|
17
|
+
constructor(options) {
|
|
18
|
+
this.pageSize = options.pageSize ?? 1000;
|
|
19
|
+
if (!Number.isInteger(this.pageSize) || this.pageSize < 1 || this.pageSize > 1000) {
|
|
20
|
+
throw new RangeError("pageSize must be an integer from 1 to 1000");
|
|
21
|
+
}
|
|
22
|
+
this.now = options.now ?? (() => new Date());
|
|
23
|
+
this.authorize = options.authorize;
|
|
24
|
+
for (const bucket of options.buckets)
|
|
25
|
+
this.buckets.set(bucket, new Map());
|
|
26
|
+
}
|
|
27
|
+
get requests() {
|
|
28
|
+
return structuredClone(this.history);
|
|
29
|
+
}
|
|
30
|
+
async begin(operation, input, options) {
|
|
31
|
+
if (options?.abortSignal?.aborted)
|
|
32
|
+
throw new S3ServiceError("AbortError", 499);
|
|
33
|
+
const request = { operation, input: structuredClone(input) };
|
|
34
|
+
this.history.push(structuredClone(request));
|
|
35
|
+
await this.authorize?.(request);
|
|
36
|
+
if (options?.abortSignal?.aborted)
|
|
37
|
+
throw new S3ServiceError("AbortError", 499);
|
|
38
|
+
if (!this.buckets.has(input.Bucket))
|
|
39
|
+
throw new S3ServiceError("NoSuchBucket", 404);
|
|
40
|
+
if ("Key" in input && (input.Key.length === 0 || Buffer.byteLength(input.Key) > 1024)) {
|
|
41
|
+
throw new S3ServiceError("InvalidArgument", 400);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
bucket(name) {
|
|
45
|
+
const bucket = this.buckets.get(name);
|
|
46
|
+
if (!bucket)
|
|
47
|
+
throw new S3ServiceError("NoSuchBucket", 404);
|
|
48
|
+
return bucket;
|
|
49
|
+
}
|
|
50
|
+
object(input) {
|
|
51
|
+
const object = this.bucket(input.Bucket).get(input.Key);
|
|
52
|
+
if (!object)
|
|
53
|
+
throw new S3ServiceError("NoSuchKey", 404);
|
|
54
|
+
return object;
|
|
55
|
+
}
|
|
56
|
+
head(object) {
|
|
57
|
+
return {
|
|
58
|
+
ContentLength: object.body.byteLength,
|
|
59
|
+
ETag: object.etag,
|
|
60
|
+
LastModified: new Date(object.modified),
|
|
61
|
+
Metadata: { ...object.metadata },
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
store(body, metadata = {}) {
|
|
65
|
+
const metadataBytes = Object.entries(metadata).reduce((total, [key, value]) => total + Buffer.byteLength(key) + Buffer.byteLength(value), 0);
|
|
66
|
+
if (metadataBytes > 2048)
|
|
67
|
+
throw new S3ServiceError("MetadataTooLarge", 400);
|
|
68
|
+
return {
|
|
69
|
+
body: new Uint8Array(body),
|
|
70
|
+
etag: `"${createHash("md5").update(body).digest("hex")}"`,
|
|
71
|
+
modified: new Date(this.now()),
|
|
72
|
+
metadata: { ...metadata },
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
async headObject(input, options) {
|
|
76
|
+
await this.begin("headObject", input, options);
|
|
77
|
+
const output = this.head(this.object(input));
|
|
78
|
+
recordMockS3Head(output, input, this.bucket(input.Bucket));
|
|
79
|
+
return output;
|
|
80
|
+
}
|
|
81
|
+
async getObject(input, options) {
|
|
82
|
+
await this.begin("getObject", input, options);
|
|
83
|
+
const object = this.object(input);
|
|
84
|
+
return { ...this.head(object), Body: new Uint8Array(object.body) };
|
|
85
|
+
}
|
|
86
|
+
async putObject(input, options) {
|
|
87
|
+
if (!(input.Body instanceof Uint8Array))
|
|
88
|
+
throw new S3ServiceError("InvalidArgument", 400);
|
|
89
|
+
input = { ...input, Body: new Uint8Array(input.Body), ...(input.Metadata ? { Metadata: { ...input.Metadata } } : {}) };
|
|
90
|
+
await this.begin("putObject", input, options);
|
|
91
|
+
const bucket = this.bucket(input.Bucket);
|
|
92
|
+
const previous = bucket.get(input.Key);
|
|
93
|
+
if (input.IfMatch !== undefined && !previous)
|
|
94
|
+
throw new S3ServiceError("NoSuchKey", 404);
|
|
95
|
+
if ((input.IfNoneMatch === "*" && previous)
|
|
96
|
+
|| (input.IfMatch !== undefined && previous?.etag !== input.IfMatch)) {
|
|
97
|
+
throw new S3ServiceError("PreconditionFailed", 412);
|
|
98
|
+
}
|
|
99
|
+
const object = this.store(input.Body, input.Metadata);
|
|
100
|
+
bucket.set(input.Key, object);
|
|
101
|
+
return { ETag: object.etag };
|
|
102
|
+
}
|
|
103
|
+
async getObjectStream(input, options) {
|
|
104
|
+
await this.begin("getObject", input, options);
|
|
105
|
+
const object = this.object(input);
|
|
106
|
+
if (input.IfMatch !== undefined && input.IfMatch !== object.etag)
|
|
107
|
+
throw new S3ServiceError("PreconditionFailed", 412);
|
|
108
|
+
let start = 0;
|
|
109
|
+
let end = object.body.length;
|
|
110
|
+
if (input.Range !== undefined) {
|
|
111
|
+
const range = /^bytes=(\d+)-(\d+)$/.exec(input.Range);
|
|
112
|
+
if (!range)
|
|
113
|
+
throw new S3ServiceError("InvalidArgument", 400);
|
|
114
|
+
start = Number(range[1]);
|
|
115
|
+
end = Math.min(Number(range[2]) + 1, end);
|
|
116
|
+
if (start >= end)
|
|
117
|
+
throw new S3ServiceError("InvalidRange", 416);
|
|
118
|
+
}
|
|
119
|
+
return {
|
|
120
|
+
...this.head(object), ContentLength: end - start,
|
|
121
|
+
Body: (async function* () {
|
|
122
|
+
for (let offset = start; offset < end; offset += 64 * 1024) {
|
|
123
|
+
if (options?.abortSignal?.aborted)
|
|
124
|
+
throw new S3ServiceError("AbortError", 499);
|
|
125
|
+
yield object.body.slice(offset, Math.min(offset + 64 * 1024, end));
|
|
126
|
+
}
|
|
127
|
+
})(),
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
async putObjectStream(input, options) {
|
|
131
|
+
const snapshot = { ...input, Body: new Uint8Array(), ...(input.Metadata ? { Metadata: { ...input.Metadata } } : {}) };
|
|
132
|
+
await this.begin("putObject", snapshot, options);
|
|
133
|
+
const body = await collectBytes(input.Body, { maxBytes: 5_000_000_000, ...(options?.abortSignal ? { signal: options.abortSignal } : {}) });
|
|
134
|
+
const bucket = this.bucket(input.Bucket);
|
|
135
|
+
const previous = bucket.get(input.Key);
|
|
136
|
+
if (input.IfMatch !== undefined && !previous)
|
|
137
|
+
throw new S3ServiceError("NoSuchKey", 404);
|
|
138
|
+
if ((input.IfNoneMatch === "*" && previous) || (input.IfMatch !== undefined && previous?.etag !== input.IfMatch)) {
|
|
139
|
+
throw new S3ServiceError("PreconditionFailed", 412);
|
|
140
|
+
}
|
|
141
|
+
const object = this.store(body, snapshot.Metadata);
|
|
142
|
+
bucket.set(input.Key, object);
|
|
143
|
+
return { ETag: object.etag };
|
|
144
|
+
}
|
|
145
|
+
async deleteObject(input, options) {
|
|
146
|
+
await this.begin("deleteObject", input, options);
|
|
147
|
+
const bucket = this.bucket(input.Bucket);
|
|
148
|
+
const previous = bucket.get(input.Key);
|
|
149
|
+
if (input.IfMatch !== undefined && !previous)
|
|
150
|
+
throw new S3ServiceError("NoSuchKey", 404);
|
|
151
|
+
if (previous && input.IfMatch !== undefined && input.IfMatch !== "*" && previous.etag !== input.IfMatch) {
|
|
152
|
+
throw new S3ServiceError("PreconditionFailed", 412);
|
|
153
|
+
}
|
|
154
|
+
bucket.delete(input.Key);
|
|
155
|
+
return {};
|
|
156
|
+
}
|
|
157
|
+
async copyObject(input, options) {
|
|
158
|
+
await this.begin("copyObject", input, options);
|
|
159
|
+
let decoded;
|
|
160
|
+
try {
|
|
161
|
+
decoded = decodeURIComponent(input.CopySource.replace(/^\//, ""));
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
throw new S3ServiceError("InvalidArgument", 400);
|
|
165
|
+
}
|
|
166
|
+
const separator = decoded.indexOf("/");
|
|
167
|
+
if (separator < 1)
|
|
168
|
+
throw new S3ServiceError("InvalidArgument", 400);
|
|
169
|
+
const source = this.object({ Bucket: decoded.slice(0, separator), Key: decoded.slice(separator + 1) });
|
|
170
|
+
if (input.CopySourceIfMatch !== undefined && input.CopySourceIfMatch !== source.etag) {
|
|
171
|
+
throw new S3ServiceError("PreconditionFailed", 412);
|
|
172
|
+
}
|
|
173
|
+
const bucket = this.bucket(input.Bucket);
|
|
174
|
+
if (input.IfNoneMatch === "*" && bucket.has(input.Key))
|
|
175
|
+
throw new S3ServiceError("PreconditionFailed", 412);
|
|
176
|
+
if (input.IfMatch !== undefined && bucket.get(input.Key)?.etag !== input.IfMatch)
|
|
177
|
+
throw new S3ServiceError("PreconditionFailed", 412);
|
|
178
|
+
const copy = this.store(source.body, input.MetadataDirective === "REPLACE" ? input.Metadata : source.metadata);
|
|
179
|
+
bucket.set(input.Key, copy);
|
|
180
|
+
return { CopyObjectResult: { ETag: copy.etag, LastModified: new Date(copy.modified) } };
|
|
181
|
+
}
|
|
182
|
+
async listObjectsV2(input, options) {
|
|
183
|
+
await this.begin("listObjectsV2", input, options);
|
|
184
|
+
const maxKeys = input.MaxKeys ?? 1000;
|
|
185
|
+
if (!Number.isInteger(maxKeys) || maxKeys < 0 || maxKeys > 1000)
|
|
186
|
+
throw new S3ServiceError("InvalidArgument", 400);
|
|
187
|
+
const prefix = input.Prefix ?? "";
|
|
188
|
+
const delimiter = input.Delimiter ?? "";
|
|
189
|
+
const scope = JSON.stringify([input.Bucket, prefix, delimiter]);
|
|
190
|
+
const cursor = input.ContinuationToken === undefined ? undefined : this.cursors.get(input.ContinuationToken);
|
|
191
|
+
if (input.ContinuationToken !== undefined && (!cursor || cursor.scope !== scope)) {
|
|
192
|
+
throw new S3ServiceError("InvalidArgument", 400, "invalid continuation token");
|
|
193
|
+
}
|
|
194
|
+
const entries = new Map();
|
|
195
|
+
for (const [key, object] of this.bucket(input.Bucket)) {
|
|
196
|
+
if (!key.startsWith(prefix))
|
|
197
|
+
continue;
|
|
198
|
+
const separator = delimiter === "" ? -1 : key.indexOf(delimiter, prefix.length);
|
|
199
|
+
if (separator >= 0) {
|
|
200
|
+
const commonPrefix = key.slice(0, separator + delimiter.length);
|
|
201
|
+
entries.set(commonPrefix, commonPrefix);
|
|
202
|
+
}
|
|
203
|
+
else {
|
|
204
|
+
entries.set(key, { Key: key, Size: object.body.byteLength, ETag: object.etag, LastModified: new Date(object.modified) });
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
const ordered = [...entries.keys()].sort(compareKeys)
|
|
208
|
+
.filter((key) => cursor === undefined || compareKeys(key, cursor.after) > 0);
|
|
209
|
+
const selected = ordered.slice(0, Math.min(maxKeys, this.pageSize));
|
|
210
|
+
const contents = [];
|
|
211
|
+
const prefixes = [];
|
|
212
|
+
for (const key of selected) {
|
|
213
|
+
const entry = entries.get(key);
|
|
214
|
+
if (typeof entry === "string")
|
|
215
|
+
prefixes.push({ Prefix: entry });
|
|
216
|
+
else if (entry)
|
|
217
|
+
contents.push(entry);
|
|
218
|
+
}
|
|
219
|
+
const truncated = maxKeys > 0 && selected.length < ordered.length;
|
|
220
|
+
let token;
|
|
221
|
+
if (truncated) {
|
|
222
|
+
token = `mock-s3-cursor-${++this.nextToken}`;
|
|
223
|
+
this.cursors.set(token, { scope, after: selected[selected.length - 1] });
|
|
224
|
+
}
|
|
225
|
+
return {
|
|
226
|
+
Contents: contents,
|
|
227
|
+
CommonPrefixes: prefixes,
|
|
228
|
+
KeyCount: selected.length,
|
|
229
|
+
IsTruncated: truncated,
|
|
230
|
+
...(token === undefined ? {} : { NextContinuationToken: token }),
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { FileStat, FileSystem, FsOptions, EntryComparison } from "../../contracts/filesystem.js";
|
|
2
|
+
import type { EntryView } from "../mount/comparison.js";
|
|
3
|
+
export interface OwnedS3Entry {
|
|
4
|
+
readonly storage: object;
|
|
5
|
+
readonly key: string;
|
|
6
|
+
}
|
|
7
|
+
export declare function recordS3Observation(filesystem: FileSystem, path: string, stat: FileStat, entry: OwnedS3Entry): void;
|
|
8
|
+
export declare function registerS3EntryOwner(filesystem: FileSystem, normalize: (path: string) => string, intact: () => boolean, baseComparison: NonNullable<FileSystem["compareEntry"]>, comparison?: NonNullable<FileSystem["compareEntry"]>): void;
|
|
9
|
+
export declare function getOwnedS3Entry(view: EntryView): OwnedS3Entry | undefined;
|
|
10
|
+
export declare function compareOwnedS3Entries(own: EntryView, peer: EntryView, options: FsOptions): Promise<EntryComparison>;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { FsError } from "../../contracts/errors.js";
|
|
2
|
+
import { assertCallbackAuthorityAllowed } from "../mount/comparison.js";
|
|
3
|
+
const observedStats = new WeakMap();
|
|
4
|
+
const entries = new WeakMap();
|
|
5
|
+
const comparisons = new WeakMap();
|
|
6
|
+
const configuredComparisons = new WeakMap();
|
|
7
|
+
export function recordS3Observation(filesystem, path, stat, entry) {
|
|
8
|
+
observedStats.set(stat, { filesystem, path, entry });
|
|
9
|
+
}
|
|
10
|
+
export function registerS3EntryOwner(filesystem, normalize, intact, baseComparison, comparison) {
|
|
11
|
+
comparisons.set(filesystem, baseComparison);
|
|
12
|
+
if (comparison)
|
|
13
|
+
configuredComparisons.set(filesystem, comparison);
|
|
14
|
+
entries.set(filesystem, view => {
|
|
15
|
+
if (!intact())
|
|
16
|
+
return undefined;
|
|
17
|
+
const observation = observedStats.get(view.stat);
|
|
18
|
+
return observation?.filesystem === filesystem && observation.path === normalize(view.path) ? observation.entry : undefined;
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
export function getOwnedS3Entry(view) {
|
|
22
|
+
return entries.get(view.filesystem)?.(view);
|
|
23
|
+
}
|
|
24
|
+
export async function compareOwnedS3Entries(own, peer, options) {
|
|
25
|
+
options.signal?.throwIfAborted();
|
|
26
|
+
let explicit = false;
|
|
27
|
+
let answer = "unknown";
|
|
28
|
+
const visited = new Set();
|
|
29
|
+
for (const [left, right] of [[own, peer], [peer, own]]) {
|
|
30
|
+
const baseComparison = comparisons.get(left.filesystem);
|
|
31
|
+
if (!baseComparison || visited.has(left.filesystem))
|
|
32
|
+
continue;
|
|
33
|
+
visited.add(left.filesystem);
|
|
34
|
+
const current = left.filesystem.compareEntry;
|
|
35
|
+
const comparison = current === baseComparison ? configuredComparisons.get(left.filesystem) : current;
|
|
36
|
+
if (current === baseComparison && comparison === undefined)
|
|
37
|
+
continue;
|
|
38
|
+
explicit = true;
|
|
39
|
+
if (comparison === undefined)
|
|
40
|
+
continue;
|
|
41
|
+
options.signal?.throwIfAborted();
|
|
42
|
+
if (typeof comparison !== "function") {
|
|
43
|
+
throw new FsError("EIO", { path: own.path, dest: peer.path, message: "invalid explicit S3 comparison method" });
|
|
44
|
+
}
|
|
45
|
+
assertCallbackAuthorityAllowed();
|
|
46
|
+
const result = await comparison.call(left.filesystem, left.path, right.filesystem, right.path, options);
|
|
47
|
+
options.signal?.throwIfAborted();
|
|
48
|
+
if (result !== "same" && result !== "distinct" && result !== "unknown") {
|
|
49
|
+
throw new FsError("EIO", { path: own.path, dest: peer.path, message: "invalid explicit S3 comparison" });
|
|
50
|
+
}
|
|
51
|
+
if (result === "unknown")
|
|
52
|
+
continue;
|
|
53
|
+
if (answer !== "unknown" && answer !== result) {
|
|
54
|
+
throw new FsError("EIO", { path: own.path, dest: peer.path, message: "conflicting explicit S3 comparisons" });
|
|
55
|
+
}
|
|
56
|
+
answer = result;
|
|
57
|
+
}
|
|
58
|
+
const left = getOwnedS3Entry(own);
|
|
59
|
+
options.signal?.throwIfAborted();
|
|
60
|
+
const right = getOwnedS3Entry(peer);
|
|
61
|
+
options.signal?.throwIfAborted();
|
|
62
|
+
const same = left && right && left.storage === right.storage && left.key === right.key;
|
|
63
|
+
if (same && answer === "distinct") {
|
|
64
|
+
throw new FsError("EIO", { path: own.path, dest: peer.path, message: "explicit S3 comparison contradicts a known alias" });
|
|
65
|
+
}
|
|
66
|
+
if (same)
|
|
67
|
+
return "same";
|
|
68
|
+
if (explicit)
|
|
69
|
+
return answer;
|
|
70
|
+
return left && right ? "distinct" : "unknown";
|
|
71
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
export interface S3RequestOptions {
|
|
2
|
+
readonly abortSignal?: AbortSignal;
|
|
3
|
+
}
|
|
4
|
+
export interface S3ObjectInput {
|
|
5
|
+
readonly Bucket: string;
|
|
6
|
+
readonly Key: string;
|
|
7
|
+
}
|
|
8
|
+
export interface S3HeadOutput {
|
|
9
|
+
readonly ContentLength?: number | undefined;
|
|
10
|
+
readonly LastModified?: Date | undefined;
|
|
11
|
+
readonly ETag?: string | undefined;
|
|
12
|
+
readonly Metadata?: Record<string, string> | undefined;
|
|
13
|
+
}
|
|
14
|
+
export type S3Body = Uint8Array | AsyncIterable<Uint8Array> | {
|
|
15
|
+
transformToByteArray(): Promise<Uint8Array>;
|
|
16
|
+
};
|
|
17
|
+
export interface S3GetOutput extends S3HeadOutput {
|
|
18
|
+
readonly Body?: S3Body | undefined;
|
|
19
|
+
}
|
|
20
|
+
export interface S3StreamGetInput extends S3ObjectInput {
|
|
21
|
+
readonly Range?: string;
|
|
22
|
+
readonly IfMatch?: string;
|
|
23
|
+
}
|
|
24
|
+
export interface S3StreamGetOutput extends S3HeadOutput {
|
|
25
|
+
readonly Body: AsyncIterable<Uint8Array>;
|
|
26
|
+
}
|
|
27
|
+
export interface S3StreamPutInput extends Omit<S3PutInput, "Body"> {
|
|
28
|
+
readonly Body: AsyncIterable<Uint8Array>;
|
|
29
|
+
}
|
|
30
|
+
export interface S3PutInput extends S3ObjectInput {
|
|
31
|
+
readonly Body: Uint8Array;
|
|
32
|
+
readonly IfMatch?: string;
|
|
33
|
+
readonly IfNoneMatch?: "*";
|
|
34
|
+
readonly Metadata?: Record<string, string>;
|
|
35
|
+
}
|
|
36
|
+
export interface S3DeleteInput extends S3ObjectInput {
|
|
37
|
+
readonly IfMatch?: string;
|
|
38
|
+
}
|
|
39
|
+
export interface S3CopyInput extends S3ObjectInput {
|
|
40
|
+
readonly CopySource: string;
|
|
41
|
+
readonly CopySourceIfMatch?: string;
|
|
42
|
+
readonly IfNoneMatch?: "*";
|
|
43
|
+
readonly IfMatch?: string;
|
|
44
|
+
readonly MetadataDirective?: "COPY" | "REPLACE";
|
|
45
|
+
readonly Metadata?: Record<string, string>;
|
|
46
|
+
}
|
|
47
|
+
export interface S3CopyOutput {
|
|
48
|
+
readonly CopyObjectResult?: {
|
|
49
|
+
readonly ETag?: string | undefined;
|
|
50
|
+
readonly LastModified?: Date | undefined;
|
|
51
|
+
} | undefined;
|
|
52
|
+
}
|
|
53
|
+
export interface S3ListInput {
|
|
54
|
+
readonly Bucket: string;
|
|
55
|
+
readonly Prefix?: string;
|
|
56
|
+
readonly Delimiter?: string;
|
|
57
|
+
readonly MaxKeys?: number;
|
|
58
|
+
readonly ContinuationToken?: string;
|
|
59
|
+
}
|
|
60
|
+
export interface S3ObjectSummary {
|
|
61
|
+
readonly Key?: string | undefined;
|
|
62
|
+
readonly Size?: number | undefined;
|
|
63
|
+
readonly ETag?: string | undefined;
|
|
64
|
+
readonly LastModified?: Date | undefined;
|
|
65
|
+
}
|
|
66
|
+
export interface S3ListOutput {
|
|
67
|
+
readonly Contents?: readonly S3ObjectSummary[] | undefined;
|
|
68
|
+
readonly CommonPrefixes?: readonly {
|
|
69
|
+
readonly Prefix?: string | undefined;
|
|
70
|
+
}[] | undefined;
|
|
71
|
+
readonly IsTruncated?: boolean | undefined;
|
|
72
|
+
readonly NextContinuationToken?: string | undefined;
|
|
73
|
+
readonly KeyCount?: number | undefined;
|
|
74
|
+
}
|
|
75
|
+
export interface S3Client {
|
|
76
|
+
getObjectStream?(input: S3StreamGetInput, options?: S3RequestOptions): Promise<S3StreamGetOutput>;
|
|
77
|
+
putObjectStream?(input: S3StreamPutInput, options?: S3RequestOptions): Promise<unknown>;
|
|
78
|
+
headObject(input: S3ObjectInput, options?: S3RequestOptions): Promise<S3HeadOutput>;
|
|
79
|
+
getObject(input: S3ObjectInput, options?: S3RequestOptions): Promise<S3GetOutput>;
|
|
80
|
+
putObject(input: S3PutInput, options?: S3RequestOptions): Promise<unknown>;
|
|
81
|
+
deleteObject(input: S3DeleteInput, options?: S3RequestOptions): Promise<unknown>;
|
|
82
|
+
copyObject(input: S3CopyInput, options?: S3RequestOptions): Promise<S3CopyOutput>;
|
|
83
|
+
listObjectsV2(input: S3ListInput, options?: S3RequestOptions): Promise<S3ListOutput>;
|
|
84
|
+
}
|
|
85
|
+
export interface S3TransportCapabilities {
|
|
86
|
+
readonly streamingRead?: boolean;
|
|
87
|
+
readonly streamingWrite?: boolean;
|
|
88
|
+
readonly conditionalPut?: boolean;
|
|
89
|
+
readonly conditionalCopy?: boolean;
|
|
90
|
+
readonly conditionalDelete?: boolean;
|
|
91
|
+
}
|
|
92
|
+
export interface S3Transport extends S3Client {
|
|
93
|
+
readonly capabilities?: S3TransportCapabilities;
|
|
94
|
+
}
|
|
95
|
+
export declare function createS3Transport(client: S3Client, capabilities?: S3TransportCapabilities): S3Transport;
|
|
96
|
+
export declare class S3ServiceError extends Error {
|
|
97
|
+
readonly code: string;
|
|
98
|
+
readonly $metadata: {
|
|
99
|
+
readonly httpStatusCode: number;
|
|
100
|
+
};
|
|
101
|
+
constructor(code: string, status: number, message?: string);
|
|
102
|
+
}
|
|
103
|
+
export declare function encodeCopySource(bucket: string, key: string): string;
|