@rdfx/fs 0.0.34 → 0.0.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/CompressedRdfFormat.d.ts +7 -0
  2. package/dist/CompressedRdfFormat.js +2 -0
  3. package/dist/CompressedRdfStream.d.ts +15 -0
  4. package/dist/CompressedRdfStream.js +68 -0
  5. package/dist/CompressionMethod.d.ts +3 -0
  6. package/dist/CompressionMethod.js +2 -0
  7. package/dist/Dirent.d.ts +8 -0
  8. package/dist/Dirent.js +2 -0
  9. package/dist/ErrnoException.d.ts +8 -0
  10. package/dist/ErrnoException.js +4 -0
  11. package/dist/FileSystem.d.ts +93 -0
  12. package/dist/FileSystem.js +2 -0
  13. package/dist/NodeFileSystem.d.ts +30 -0
  14. package/dist/NodeFileSystem.js +42 -0
  15. package/dist/RdfDirectory.d.ts +12 -4
  16. package/dist/RdfDirectory.js +37 -50
  17. package/dist/RdfDirectoryGraphStore.d.ts +7 -3
  18. package/dist/RdfDirectoryGraphStore.js +58 -70
  19. package/dist/RdfFile.d.ts +19 -20
  20. package/dist/RdfFile.js +29 -104
  21. package/dist/RdfFileGraphStore.d.ts +9 -5
  22. package/dist/RdfFileGraphStore.js +35 -39
  23. package/dist/RdfFileSystemEntry.d.ts +3 -1
  24. package/dist/RdfFileSystemEntry.js +3 -2
  25. package/dist/RdfFormat.d.ts +8 -2
  26. package/dist/RdfFormat.js +47 -9
  27. package/dist/Stat.d.ts +9 -0
  28. package/dist/Stat.js +2 -0
  29. package/dist/UncompressedRdfFormat.d.ts +7 -0
  30. package/dist/UncompressedRdfFormat.js +2 -0
  31. package/dist/compressionMethods.d.ts +2 -0
  32. package/dist/compressionMethods.js +6 -0
  33. package/dist/index.d.ts +12 -1
  34. package/dist/index.js +12 -1
  35. package/dist/uncompressedRdfFormats.d.ts +3 -0
  36. package/dist/uncompressedRdfFormats.js +33 -0
  37. package/dist/uncompressedRdfFormatsByMimeType.d.ts +3 -0
  38. package/dist/uncompressedRdfFormatsByMimeType.js +6 -0
  39. package/package.json +7 -6
  40. package/dist/AbstractRdfFileSystemEntry.d.ts +0 -19
  41. package/dist/AbstractRdfFileSystemEntry.js +0 -11
  42. package/dist/stat.d.ts +0 -4
  43. package/dist/stat.js +0 -11
@@ -0,0 +1,7 @@
1
+ import type { CompressionMethod } from "./CompressionMethod.js";
2
+ import type { UncompressedRdfFormat } from "./UncompressedRdfFormat.js";
3
+ export interface CompressedRdfFormat extends Omit<UncompressedRdfFormat, "mimeType"> {
4
+ readonly mimeType: CompressionMethod;
5
+ readonly uncompressedMimeType: UncompressedRdfFormat["mimeType"];
6
+ }
7
+ //# sourceMappingURL=CompressedRdfFormat.d.ts.map
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=CompressedRdfFormat.js.map
@@ -0,0 +1,15 @@
1
+ import type { Readable, Writable } from "node:stream";
2
+ import type { Stream } from "@rdfjs/types";
3
+ import serializers from "@rdfx/serializers";
4
+ import { type Either } from "purify-ts";
5
+ import { RdfFormat } from "./RdfFormat.js";
6
+ export declare namespace CompressedRdfStream {
7
+ function parse(format: RdfFormat, input: Readable): Stream;
8
+ function serialize({ destination, format, serializerOptions, source, }: {
9
+ destination: Writable;
10
+ format: RdfFormat;
11
+ serializerOptions?: Parameters<typeof serializers>[0];
12
+ source: Stream;
13
+ }): Promise<Either<Error, void>>;
14
+ }
15
+ //# sourceMappingURL=CompressedRdfStream.d.ts.map
@@ -0,0 +1,68 @@
1
+ import { pipeline } from "node:stream/promises";
2
+ import zlib from "node:zlib";
3
+ import dataFactory from "@rdfx/data-factory";
4
+ import parsersFactory from "@rdfx/parsers";
5
+ import serializers from "@rdfx/serializers";
6
+ import { EitherAsync } from "purify-ts";
7
+ import bz2 from "unbzip2-stream";
8
+ import { RdfFormat } from "./RdfFormat.js";
9
+ const parsers = parsersFactory({ dataFactory });
10
+ export var CompressedRdfStream;
11
+ (function (CompressedRdfStream) {
12
+ function parse(format, input) {
13
+ let uncompressedMimeType;
14
+ if (RdfFormat.isCompressed(format)) {
15
+ switch (format.mimeType) {
16
+ case "application/gzip":
17
+ input = input.pipe(zlib.createGunzip());
18
+ break;
19
+ case "application/x-brotli":
20
+ input = input.pipe(zlib.createBrotliDecompress());
21
+ break;
22
+ case "application/x-bzip2":
23
+ input = input.pipe(bz2());
24
+ break;
25
+ }
26
+ uncompressedMimeType = format.uncompressedMimeType;
27
+ }
28
+ else {
29
+ uncompressedMimeType = format.mimeType;
30
+ }
31
+ const output = parsers.import(uncompressedMimeType, input);
32
+ if (output === null) {
33
+ throw new RangeError(`unsupported RDF serialization format: ${uncompressedMimeType}`);
34
+ }
35
+ return output;
36
+ }
37
+ CompressedRdfStream.parse = parse;
38
+ async function serialize({ destination, format, serializerOptions, source, }) {
39
+ return EitherAsync(async () => {
40
+ const uncompressedMimeType = RdfFormat.isCompressed(format)
41
+ ? format.uncompressedMimeType
42
+ : format.mimeType;
43
+ const rdfStream = serializers(serializerOptions).import(uncompressedMimeType, source);
44
+ if (rdfStream === null) {
45
+ throw new RangeError(`unsupported RDF serialization format: ${uncompressedMimeType}`);
46
+ }
47
+ if (RdfFormat.isCompressed(format)) {
48
+ let compressor;
49
+ switch (format.mimeType) {
50
+ case "application/gzip":
51
+ compressor = zlib.createGzip();
52
+ break;
53
+ case "application/x-brotli":
54
+ compressor = zlib.createBrotliCompress();
55
+ break;
56
+ case "application/x-bzip2":
57
+ throw new RangeError("bzip2 compression unsupported");
58
+ }
59
+ await pipeline(rdfStream, compressor, destination);
60
+ }
61
+ else {
62
+ await pipeline(rdfStream, destination);
63
+ }
64
+ });
65
+ }
66
+ CompressedRdfStream.serialize = serialize;
67
+ })(CompressedRdfStream || (CompressedRdfStream = {}));
68
+ //# sourceMappingURL=CompressedRdfStream.js.map
@@ -0,0 +1,3 @@
1
+ import type { compressionMethods } from "./compressionMethods.js";
2
+ export type CompressionMethod = (typeof compressionMethods)[number];
3
+ //# sourceMappingURL=CompressionMethod.d.ts.map
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=CompressionMethod.js.map
@@ -0,0 +1,8 @@
1
+ export interface Dirent {
2
+ readonly name: string;
3
+ readonly parentPath: string;
4
+ isDirectory(): boolean;
5
+ isFile(): boolean;
6
+ isSymbolicLink(): boolean;
7
+ }
8
+ //# sourceMappingURL=Dirent.d.ts.map
package/dist/Dirent.js ADDED
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=Dirent.js.map
@@ -0,0 +1,8 @@
1
+ export interface ErrnoException extends Error {
2
+ code?: string;
3
+ errno?: number;
4
+ path?: string;
5
+ syscall?: string;
6
+ }
7
+ export declare function isErrnoException(error: unknown): error is ErrnoException;
8
+ //# sourceMappingURL=ErrnoException.d.ts.map
@@ -0,0 +1,4 @@
1
+ export function isErrnoException(error) {
2
+ return error instanceof Error && ("code" in error || "errno" in error);
3
+ }
4
+ //# sourceMappingURL=ErrnoException.js.map
@@ -0,0 +1,93 @@
1
+ import type { Readable, Writable } from "node:stream";
2
+ import type { Either } from "purify-ts";
3
+ import type { Dirent } from "./Dirent.js";
4
+ import type { ErrnoException } from "./ErrnoException.js";
5
+ import type { Stat } from "./Stat.js";
6
+ /**
7
+ * Promise-based abstraction over node:fs.
8
+ */
9
+ export interface FileSystem {
10
+ /**
11
+ * Create a directory (mkdir).
12
+ *
13
+ * @param path path to the new directory
14
+ * @param recursive: create any parent directories as necessary
15
+ */
16
+ createDirectory(path: string, options?: FileSystem.CreateDirectoryOptions): Promise<Either<ErrnoException, void>>;
17
+ /**
18
+ * Delete a directory.
19
+ *
20
+ * @param path path to the directory
21
+ * @param force ignore ENOENT errors
22
+ * @param recursive: recursively delete files and subdirectories in the directory
23
+ */
24
+ deleteDirectory(path: string, options?: FileSystem.DeleteDirectoryOptions): Promise<Either<ErrnoException, void>>;
25
+ /**
26
+ * Delete a file (unlink).
27
+ *
28
+ * @param path path to the file
29
+ * @param force ignore ENOENT errors
30
+ */
31
+ deleteFile(path: string, options?: FileSystem.DeleteFileOptions): Promise<Either<ErrnoException, void>>;
32
+ /**
33
+ * Read/list entries in the directory.
34
+ *
35
+ * @param path path to the directory
36
+ * @param recursive recursively iterate directory entries
37
+ */
38
+ readDirectory(path: string, options?: FileSystem.ReadDirectoryOptions): Promise<Either<ErrnoException, readonly Dirent[]>>;
39
+ /**
40
+ * Read a file completely, returning its data as a byte array.
41
+ *
42
+ * @param path path to the file
43
+ */
44
+ readFile(path: string): Promise<Either<ErrnoException, Uint8Array>>;
45
+ /**
46
+ * Read a file with a stream.
47
+ *
48
+ * @param path path to the file
49
+ */
50
+ createReadStream(path: string): Readable;
51
+ /**
52
+ * Get the real path of a symbolic link.
53
+ *
54
+ * @param path path to the symbolic link
55
+ */
56
+ realpath(path: string): Promise<Either<ErrnoException, string>>;
57
+ /**
58
+ * Get metadata about a file or directory.
59
+ *
60
+ * @param path path to the file or directory
61
+ */
62
+ stat(path: string): Promise<Either<ErrnoException, Stat>>;
63
+ /**
64
+ * Overwrite a file.
65
+ *
66
+ * @param path path to the file.
67
+ * @param data data to write
68
+ */
69
+ writeFile(path: string, data: Uint8Array): Promise<Either<ErrnoException, void>>;
70
+ /**
71
+ * Overwrite a file by borrowing and using a stream.
72
+ *
73
+ * @param path path to the file
74
+ * @param write function that writes to the stream and only returns once writing is done
75
+ */
76
+ writeFileStream<ReturnT>(path: string, write: (stream: Writable) => Promise<Either<Error, ReturnT>>): Promise<Either<Error, ReturnT>>;
77
+ }
78
+ export declare namespace FileSystem {
79
+ type CreateDirectoryOptions = Readonly<{
80
+ recursive?: boolean;
81
+ }>;
82
+ type DeleteDirectoryOptions = Readonly<{
83
+ force?: boolean;
84
+ recursive?: boolean;
85
+ }>;
86
+ type DeleteFileOptions = Readonly<{
87
+ force?: boolean;
88
+ }>;
89
+ type ReadDirectoryOptions = Readonly<{
90
+ recursive?: boolean;
91
+ }>;
92
+ }
93
+ //# sourceMappingURL=FileSystem.d.ts.map
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=FileSystem.js.map
@@ -0,0 +1,30 @@
1
+ import type { Readable, Writable } from "node:stream";
2
+ import { type Either } from "purify-ts";
3
+ import type { Dirent } from "./Dirent.js";
4
+ import type { ErrnoException } from "./ErrnoException.js";
5
+ import type { FileSystem } from "./FileSystem.js";
6
+ import type { Stat } from "./Stat.js";
7
+ export declare class NodeFileSystem implements FileSystem {
8
+ static instance: NodeFileSystem;
9
+ protected constructor();
10
+ createDirectory(path: string, options?: {
11
+ recursive?: boolean;
12
+ }): Promise<Either<ErrnoException, void>>;
13
+ createReadStream(path: string): Readable;
14
+ deleteDirectory(path: string, options?: {
15
+ force?: boolean;
16
+ recursive?: boolean;
17
+ }): Promise<Either<ErrnoException, void>>;
18
+ deleteFile(path: string, options?: {
19
+ force?: boolean;
20
+ }): Promise<Either<ErrnoException, void>>;
21
+ readDirectory(path: string, options?: {
22
+ recursive?: boolean;
23
+ }): Promise<Either<ErrnoException, readonly Dirent[]>>;
24
+ readFile(path: string): Promise<Either<ErrnoException, Uint8Array>>;
25
+ realpath(path: string): Promise<Either<ErrnoException, string>>;
26
+ stat(path: string): Promise<Either<ErrnoException, Stat>>;
27
+ writeFile(path: string, data: Uint8Array): Promise<Either<ErrnoException, void>>;
28
+ writeFileStream<ReturnT>(path: string, write: (stream: Writable) => Promise<Either<Error, ReturnT>>): Promise<Either<Error, ReturnT>>;
29
+ }
30
+ //# sourceMappingURL=NodeFileSystem.d.ts.map
@@ -0,0 +1,42 @@
1
+ import * as fs from "node:fs";
2
+ import { EitherAsync } from "purify-ts";
3
+ export class NodeFileSystem {
4
+ static instance = new NodeFileSystem();
5
+ constructor() { }
6
+ async createDirectory(path, options) {
7
+ return EitherAsync(async () => {
8
+ await fs.promises.mkdir(path, options);
9
+ });
10
+ }
11
+ createReadStream(path) {
12
+ return fs.createReadStream(path);
13
+ }
14
+ async deleteDirectory(path, options) {
15
+ return EitherAsync(() => fs.promises.rm(path, options));
16
+ }
17
+ async deleteFile(path, options) {
18
+ return EitherAsync(() => fs.promises.rm(path, options));
19
+ }
20
+ async readDirectory(path, options) {
21
+ return EitherAsync(() => fs.promises.readdir(path, { ...options, withFileTypes: true }));
22
+ }
23
+ async readFile(path) {
24
+ return EitherAsync(() => fs.promises.readFile(path));
25
+ }
26
+ async realpath(path) {
27
+ return EitherAsync(() => fs.promises.realpath(path));
28
+ }
29
+ async stat(path) {
30
+ return EitherAsync(() => fs.promises.stat(path));
31
+ }
32
+ async writeFile(path, data) {
33
+ return EitherAsync(() => fs.promises.writeFile(path, data));
34
+ }
35
+ async writeFileStream(path, write) {
36
+ return EitherAsync(async ({ liftEither }) => {
37
+ const stream = fs.createWriteStream(path);
38
+ return await liftEither(await write(stream));
39
+ });
40
+ }
41
+ }
42
+ //# sourceMappingURL=NodeFileSystem.js.map
@@ -1,18 +1,26 @@
1
1
  import type PrefixMap from "@rdfjs/prefix-map/PrefixMap.js";
2
- import type { DatasetCore, Quad, Stream } from "@rdfjs/types";
2
+ import type { DatasetCore, Stream } from "@rdfjs/types";
3
3
  import { type Either } from "purify-ts";
4
- import { AbstractRdfFileSystemEntry } from "./AbstractRdfFileSystemEntry.js";
4
+ import { type Logger } from "ts-log";
5
+ import type { FileSystem } from "./FileSystem.js";
5
6
  import { RdfFile } from "./RdfFile.js";
6
7
  /**
7
8
  * Abstraction for iterating over a directory of files with RDF data in them.
8
9
  */
9
- export declare class RdfDirectory extends AbstractRdfFileSystemEntry {
10
+ export declare class RdfDirectory {
11
+ readonly path: string;
12
+ private readonly fileSystem;
13
+ private readonly logger;
14
+ constructor(path: string, options?: {
15
+ fileSystem?: FileSystem;
16
+ logger?: Logger;
17
+ });
10
18
  files(options?: {
11
19
  recursive?: boolean;
12
20
  }): AsyncGenerator<RdfFile>;
13
21
  parse(options?: {
14
22
  recursive?: boolean;
15
- }): Stream<Quad>;
23
+ }): Stream;
16
24
  parseInto(dataset: DatasetCore, options?: {
17
25
  prefixMap?: PrefixMap;
18
26
  recursive?: boolean;
@@ -1,55 +1,23 @@
1
- import fs from "node:fs/promises";
2
1
  import path from "node:path";
3
2
  import { Readable } from "node:stream";
4
3
  import { EitherAsync } from "purify-ts";
5
- import { AbstractRdfFileSystemEntry } from "./AbstractRdfFileSystemEntry.js";
4
+ import { dummyLogger } from "ts-log";
5
+ import { NodeFileSystem } from "./NodeFileSystem.js";
6
6
  import { RdfFile } from "./RdfFile.js";
7
- import { stat } from "./stat.js";
8
7
  /**
9
8
  * Abstraction for iterating over a directory of files with RDF data in them.
10
9
  */
11
- export class RdfDirectory extends AbstractRdfFileSystemEntry {
10
+ export class RdfDirectory {
11
+ path;
12
+ fileSystem;
13
+ logger;
14
+ constructor(path, options) {
15
+ this.path = path;
16
+ this.fileSystem = options?.fileSystem ?? NodeFileSystem.instance;
17
+ this.logger = options?.logger ?? dummyLogger;
18
+ }
12
19
  async *files(options) {
13
- const logger = this.logger;
14
- const recursive = !!options?.recursive;
15
- async function* visitDirectory(directoryPath) {
16
- for (const dirent of await fs.readdir(directoryPath, {
17
- withFileTypes: true,
18
- })) {
19
- if (dirent.name.startsWith(".")) {
20
- continue;
21
- }
22
- let direntPath = path.resolve(directoryPath, dirent.name);
23
- let direntIsDirectory;
24
- let direntIsFile;
25
- if (dirent.isSymbolicLink()) {
26
- direntPath = await fs.realpath(direntPath);
27
- const stat = await fs.stat(direntPath);
28
- direntIsDirectory = stat.isDirectory();
29
- direntIsFile = stat.isFile();
30
- }
31
- else {
32
- direntIsDirectory = dirent.isDirectory();
33
- direntIsFile = dirent.isFile();
34
- }
35
- if (direntIsDirectory && recursive) {
36
- yield* visitDirectory(direntPath);
37
- }
38
- else if (direntIsFile) {
39
- yield* visitFile(direntPath);
40
- }
41
- else {
42
- logger.warn("%s is not a directory or file", direntPath);
43
- }
44
- }
45
- }
46
- async function* visitFile(filePath) {
47
- const file = RdfFile.fromPath(filePath, { logger });
48
- if (file.isRight()) {
49
- yield file.extract();
50
- }
51
- }
52
- const statEither = await stat(this.path);
20
+ const statEither = await this.fileSystem.stat(this.path);
53
21
  if (statEither.isLeft()) {
54
22
  this.logger.debug("%s does not exist or is not accessible: %s", this.path, statEither.extract().message);
55
23
  return;
@@ -57,14 +25,33 @@ export class RdfDirectory extends AbstractRdfFileSystemEntry {
57
25
  let stat_ = statEither.unsafeCoerce();
58
26
  let thisPath = this.path;
59
27
  if (stat_.isSymbolicLink()) {
60
- thisPath = await fs.realpath(this.path);
61
- stat_ = await fs.stat(thisPath);
62
- }
63
- if (stat_.isDirectory()) {
64
- yield* visitDirectory(thisPath);
28
+ thisPath = (await this.fileSystem.realpath(this.path)).unsafeCoerce();
29
+ stat_ = (await this.fileSystem.stat(thisPath)).unsafeCoerce();
65
30
  }
66
- else {
31
+ if (!stat_.isDirectory()) {
67
32
  this.logger.warn("%s is not an (RDF) directory", this.path);
33
+ return;
34
+ }
35
+ for (const dirent of (await this.fileSystem.readDirectory(this.path, options)).unsafeCoerce()) {
36
+ if (dirent.name.startsWith(".")) {
37
+ continue;
38
+ }
39
+ let direntPath = path.resolve(dirent.parentPath, dirent.name);
40
+ let direntIsFile;
41
+ if (dirent.isSymbolicLink()) {
42
+ direntPath = (await this.fileSystem.realpath(direntPath)).unsafeCoerce();
43
+ const stat = (await this.fileSystem.stat(direntPath)).unsafeCoerce();
44
+ direntIsFile = stat.isFile();
45
+ }
46
+ else {
47
+ direntIsFile = dirent.isFile();
48
+ }
49
+ if (direntIsFile) {
50
+ const file = RdfFile.fromPath(direntPath, { logger: this.logger });
51
+ if (file.isRight()) {
52
+ yield file.extract();
53
+ }
54
+ }
68
55
  }
69
56
  }
70
57
  parse(options) {
@@ -2,18 +2,23 @@ import type { DatasetCore, Stream } from "@rdfjs/types";
2
2
  import { GraphIdentifier, type GraphStore } from "@rdfx/graph-store";
3
3
  import { Either, Maybe } from "purify-ts";
4
4
  import { type Logger } from "ts-log";
5
+ import type { FileSystem } from "./FileSystem.js";
5
6
  /**
6
7
  * A GraphStore implementation backed by files in a directory.
7
8
  */
8
9
  export declare class RdfDirectoryGraphStore implements GraphStore {
9
- readonly directoryPath: string;
10
+ readonly path: string;
11
+ private readonly fileSystem;
10
12
  private readonly logger;
11
- constructor(directoryPath: string, options?: {
13
+ static readonly fileFormat: import("./UncompressedRdfFormat.js").UncompressedRdfFormat;
14
+ constructor(path: string, options?: {
15
+ fileSystem?: FileSystem;
12
16
  logger?: Logger;
13
17
  });
14
18
  clear(): Promise<Either<Error, object>>;
15
19
  delete(identifier: GraphIdentifier): Promise<Either<Error, object>>;
16
20
  get(identifier: GraphIdentifier): Promise<Either<Error, Maybe<Stream>>>;
21
+ graphFilePath(identifier: GraphIdentifier | string): string;
17
22
  head(identifier: GraphIdentifier): Promise<Either<Error, boolean>>;
18
23
  identifiers(): Promise<Either<Error, readonly GraphIdentifier[]>>;
19
24
  isEmpty(): Promise<Either<Error, boolean>>;
@@ -21,7 +26,6 @@ export declare class RdfDirectoryGraphStore implements GraphStore {
21
26
  put(quads: Stream): Promise<Either<Error, object>>;
22
27
  unionDataset(): Promise<Either<Error, DatasetCore>>;
23
28
  private files;
24
- private graphFilePath;
25
29
  private postOrPut;
26
30
  }
27
31
  //# sourceMappingURL=RdfDirectoryGraphStore.d.ts.map