@rdfx/fs 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,18 @@
1
+ import type { DatasetCore, Quad, Stream } from "@rdfjs/types";
2
+ import type { Either } from "purify-ts";
3
+ import { type Logger } from "ts-log";
4
+ export declare abstract class AbstractRdfFileSystemEntry {
5
+ protected readonly logger: Logger;
6
+ readonly path: string;
7
+ constructor({ logger, path }: {
8
+ logger?: Logger;
9
+ path: string;
10
+ });
11
+ abstract parse(options?: {
12
+ recursive?: boolean;
13
+ }): Stream<Quad>;
14
+ abstract parseInto(dataset: DatasetCore, options?: {
15
+ recursive?: boolean;
16
+ }): Promise<Either<Error, DatasetCore>>;
17
+ }
18
+ //# sourceMappingURL=AbstractRdfFileSystemEntry.d.ts.map
@@ -0,0 +1,8 @@
1
+ import { dummyLogger } from "ts-log";
2
+ export class AbstractRdfFileSystemEntry {
3
+ constructor({ logger, path }) {
4
+ this.logger = logger ?? dummyLogger;
5
+ this.path = path;
6
+ }
7
+ }
8
+ //# sourceMappingURL=AbstractRdfFileSystemEntry.js.map
@@ -0,0 +1,19 @@
1
+ import type { DatasetCore, Quad, Stream } from "@rdfjs/types";
2
+ import { type Either } from "purify-ts";
3
+ import { AbstractRdfFileSystemEntry } from "./AbstractRdfFileSystemEntry.js";
4
+ import { RdfFile } from "./RdfFile.js";
5
+ /**
6
+ * Abstraction for iterating over a directory of files with RDF data in them.
7
+ */
8
+ export declare class RdfDirectory extends AbstractRdfFileSystemEntry {
9
+ files(options?: {
10
+ recursive?: boolean;
11
+ }): AsyncGenerator<RdfFile>;
12
+ parse(options?: {
13
+ recursive?: boolean;
14
+ }): Stream<Quad>;
15
+ parseInto(dataset: DatasetCore, options?: {
16
+ recursive?: boolean;
17
+ }): Promise<Either<Error, DatasetCore>>;
18
+ }
19
+ //# sourceMappingURL=RdfDirectory.d.ts.map
@@ -0,0 +1,92 @@
1
+ import fs from "node:fs/promises";
2
+ import path from "node:path";
3
+ import { Readable } from "node:stream";
4
+ import { EitherAsync } from "purify-ts";
5
+ import { AbstractRdfFileSystemEntry } from "./AbstractRdfFileSystemEntry.js";
6
+ import { RdfFile } from "./RdfFile.js";
7
+ import { stat } from "./stat.js";
8
+ /**
9
+ * Abstraction for iterating over a directory of files with RDF data in them.
10
+ */
11
+ export class RdfDirectory extends AbstractRdfFileSystemEntry {
12
+ async *files(options) {
13
+ const logger = this.logger;
14
+ const recursive = !!options?.recursive;
15
+ async function* visitDirectory(directoryPath) {
16
+ for (const dirent of await fs.readdir(directoryPath, {
17
+ withFileTypes: true,
18
+ })) {
19
+ if (dirent.name.startsWith(".")) {
20
+ continue;
21
+ }
22
+ let direntPath = path.resolve(directoryPath, dirent.name);
23
+ let direntIsDirectory;
24
+ let direntIsFile;
25
+ if (dirent.isSymbolicLink()) {
26
+ direntPath = await fs.realpath(direntPath);
27
+ const stat = await fs.stat(direntPath);
28
+ direntIsDirectory = stat.isDirectory();
29
+ direntIsFile = stat.isFile();
30
+ }
31
+ else {
32
+ direntIsDirectory = dirent.isDirectory();
33
+ direntIsFile = dirent.isFile();
34
+ }
35
+ if (direntIsDirectory && recursive) {
36
+ yield* visitDirectory(direntPath);
37
+ }
38
+ else if (direntIsFile) {
39
+ yield* visitFile(direntPath);
40
+ }
41
+ else {
42
+ logger.warn("%s is not a directory or file", direntPath);
43
+ }
44
+ }
45
+ }
46
+ async function* visitFile(filePath) {
47
+ const file = RdfFile.fromPath(filePath, { logger });
48
+ if (file.isRight()) {
49
+ yield file.extract();
50
+ }
51
+ }
52
+ const statEither = await stat(this.path);
53
+ if (statEither.isLeft()) {
54
+ this.logger.debug("%s does not exist or is not accessible: %s", this.path, statEither.extract().message);
55
+ return;
56
+ }
57
+ let stat_ = statEither.unsafeCoerce();
58
+ let thisPath = this.path;
59
+ if (stat_.isSymbolicLink()) {
60
+ thisPath = await fs.realpath(this.path);
61
+ stat_ = await fs.stat(thisPath);
62
+ }
63
+ if (stat_.isDirectory()) {
64
+ yield* visitDirectory(thisPath);
65
+ }
66
+ else {
67
+ this.logger.warn("%s is not an (RDF) directory", this.path);
68
+ }
69
+ }
70
+ parse(options) {
71
+ const self = this;
72
+ async function* parseFiles() {
73
+ for await (const file of self.files(options)) {
74
+ for await (const quad of file.parse()) {
75
+ yield quad;
76
+ }
77
+ }
78
+ }
79
+ return Readable.from(parseFiles(), {
80
+ objectMode: true,
81
+ });
82
+ }
83
+ async parseInto(dataset, options) {
84
+ return EitherAsync(async ({ liftEither }) => {
85
+ for await (const file of this.files(options)) {
86
+ await liftEither(await file.parseInto(dataset));
87
+ }
88
+ return dataset;
89
+ });
90
+ }
91
+ }
92
+ //# sourceMappingURL=RdfDirectory.js.map
@@ -0,0 +1,25 @@
1
+ import type { DatasetCore, Quad, Stream } from "@rdfjs/types";
2
+ import { Either, type Maybe } from "purify-ts";
3
+ import type { Logger } from "ts-log";
4
+ import { AbstractRdfFileSystemEntry } from "./AbstractRdfFileSystemEntry.js";
5
+ import { type RdfFormat } from "./RdfFormat.js";
6
+ export declare class RdfFile extends AbstractRdfFileSystemEntry {
7
+ readonly format: RdfFile.Format;
8
+ constructor({ format, ...superParameters }: {
9
+ format: RdfFile.Format;
10
+ } & ConstructorParameters<typeof AbstractRdfFileSystemEntry>[0]);
11
+ static fromPath(filePath: string, options?: {
12
+ logger?: Logger;
13
+ }): Either<Error, RdfFile>;
14
+ parse(): Stream<Quad>;
15
+ parseInto(dataset: DatasetCore): Promise<Either<Error, DatasetCore>>;
16
+ }
17
+ export declare namespace RdfFile {
18
+ const COMPRESSION_METHODS: readonly ["application/gzip", "application/x-brotli", "application/x-bzip2"];
19
+ type CompressionMethod = (typeof COMPRESSION_METHODS)[number];
20
+ interface Format {
21
+ compressionMethod: Maybe<CompressionMethod>;
22
+ rdfFormat: RdfFormat;
23
+ }
24
+ }
25
+ //# sourceMappingURL=RdfFile.d.ts.map
@@ -0,0 +1,93 @@
1
+ import fs from "node:fs";
2
+ import * as path from "node:path";
3
+ import zlib from "node:zlib";
4
+ import formats from "@rdfjs/formats";
5
+ import { Mime } from "mime";
6
+ import otherMimeTypes from "mime/types/other.js";
7
+ import standardMimeTypes from "mime/types/standard.js";
8
+ import { Either, Just, Left, Nothing, Right } from "purify-ts";
9
+ import bz2 from "unbzip2-stream";
10
+ import { AbstractRdfFileSystemEntry } from "./AbstractRdfFileSystemEntry.js";
11
+ import { RDF_FORMATS } from "./RdfFormat.js";
12
+ const mime = new Mime(standardMimeTypes, otherMimeTypes, {
13
+ "application/x-brotli": ["br"],
14
+ });
15
+ export class RdfFile extends AbstractRdfFileSystemEntry {
16
+ constructor({ format, ...superParameters }) {
17
+ super(superParameters);
18
+ this.format = format;
19
+ }
20
+ static fromPath(filePath, options) {
21
+ const mimeType = mime.getType(filePath);
22
+ if (mimeType === null) {
23
+ return Left(new Error(`unable to infer MIME type of ${filePath}`));
24
+ }
25
+ for (const compressionMethod of RdfFile.COMPRESSION_METHODS) {
26
+ if (compressionMethod === mimeType) {
27
+ const uncompressedFileName = path.basename(path.basename(filePath), path.extname(filePath));
28
+ const uncompressedMimeType = mime.getType(uncompressedFileName);
29
+ if (uncompressedMimeType === null) {
30
+ return Left(new Error(`unable to infer MIME type of ${uncompressedFileName}`));
31
+ }
32
+ for (const rdfFormat of RDF_FORMATS) {
33
+ if (uncompressedMimeType === rdfFormat) {
34
+ return Right(new RdfFile({
35
+ format: {
36
+ compressionMethod: Just(compressionMethod),
37
+ rdfFormat,
38
+ },
39
+ logger: options?.logger,
40
+ path: filePath,
41
+ }));
42
+ }
43
+ }
44
+ }
45
+ }
46
+ for (const rdfFormat of RDF_FORMATS) {
47
+ if (mimeType === rdfFormat) {
48
+ return Right(new RdfFile({
49
+ format: {
50
+ compressionMethod: Nothing,
51
+ rdfFormat,
52
+ },
53
+ logger: options?.logger,
54
+ path: filePath,
55
+ }));
56
+ }
57
+ }
58
+ return Left(new Error(`${filePath} has a non-RDF MIME type: ${mimeType}`));
59
+ }
60
+ parse() {
61
+ let rdfFileStream = fs.createReadStream(this.path);
62
+ if (this.format.compressionMethod.isJust()) {
63
+ switch (this.format.compressionMethod.unsafeCoerce()) {
64
+ case "application/gzip":
65
+ rdfFileStream = rdfFileStream.pipe(zlib.createGunzip());
66
+ break;
67
+ case "application/x-brotli":
68
+ rdfFileStream = rdfFileStream.pipe(zlib.createBrotliDecompress());
69
+ break;
70
+ case "application/x-bzip2":
71
+ rdfFileStream = rdfFileStream.pipe(bz2());
72
+ break;
73
+ }
74
+ }
75
+ return formats.parsers.import(this.format.rdfFormat, rdfFileStream);
76
+ }
77
+ parseInto(dataset) {
78
+ return new Promise((resolve) => {
79
+ const stream = this.parse();
80
+ stream.on("data", (quad) => dataset.add(quad));
81
+ stream.on("end", () => resolve(Either.of(dataset)));
82
+ stream.on("error", (error) => resolve(Left(error)));
83
+ });
84
+ }
85
+ }
86
+ (function (RdfFile) {
87
+ RdfFile.COMPRESSION_METHODS = [
88
+ "application/gzip",
89
+ "application/x-brotli",
90
+ "application/x-bzip2",
91
+ ];
92
+ })(RdfFile || (RdfFile = {}));
93
+ //# sourceMappingURL=RdfFile.js.map
@@ -0,0 +1,11 @@
1
+ import { type Either } from "purify-ts";
2
+ import type { Logger } from "ts-log";
3
+ import { RdfDirectory } from "./RdfDirectory.js";
4
+ import { RdfFile } from "./RdfFile.js";
5
+ export type RdfFileSystemEntry = RdfDirectory | RdfFile;
6
+ export declare namespace RdfFileSystemEntry {
7
+ function fromPath(path: string, options?: {
8
+ logger: Logger;
9
+ }): Promise<Either<Error, RdfFileSystemEntry>>;
10
+ }
11
+ //# sourceMappingURL=RdfFileSystemEntry.d.ts.map
@@ -0,0 +1,20 @@
1
+ import { EitherAsync } from "purify-ts";
2
+ import { RdfDirectory } from "./RdfDirectory.js";
3
+ import { RdfFile } from "./RdfFile.js";
4
+ import { stat } from "./stat.js";
5
+ export var RdfFileSystemEntry;
6
+ (function (RdfFileSystemEntry) {
7
+ async function fromPath(path, options) {
8
+ return EitherAsync(async ({ liftEither }) => {
9
+ const stat_ = await liftEither(await stat(path));
10
+ if (stat_.isDirectory())
11
+ return new RdfDirectory({ logger: options?.logger, path });
12
+ if (stat_.isFile()) {
13
+ return await liftEither(RdfFile.fromPath(path, { logger: options?.logger }));
14
+ }
15
+ throw new Error(`${path} is not a directory or a file`);
16
+ });
17
+ }
18
+ RdfFileSystemEntry.fromPath = fromPath;
19
+ })(RdfFileSystemEntry || (RdfFileSystemEntry = {}));
20
+ //# sourceMappingURL=RdfFileSystemEntry.js.map
@@ -0,0 +1,3 @@
1
+ export declare const RDF_FORMATS: readonly ["application/ld+json", "application/n-quads", "application/n-triples", "application/rdf+xml", "application/trig", "text/n3", "text/turtle"];
2
+ export type RdfFormat = (typeof RDF_FORMATS)[number];
3
+ //# sourceMappingURL=RdfFormat.d.ts.map
@@ -0,0 +1,10 @@
1
+ export const RDF_FORMATS = [
2
+ "application/ld+json",
3
+ "application/n-quads",
4
+ "application/n-triples",
5
+ "application/rdf+xml",
6
+ "application/trig",
7
+ "text/n3",
8
+ "text/turtle",
9
+ ];
10
+ //# sourceMappingURL=RdfFormat.js.map
@@ -0,0 +1,2 @@
1
+ export * from "./stat.js";
2
+ //# sourceMappingURL=index.d.ts.map
package/dist/index.js ADDED
@@ -0,0 +1,2 @@
1
+ export * from "./stat.js";
2
+ //# sourceMappingURL=index.js.map
package/dist/stat.d.ts ADDED
@@ -0,0 +1,4 @@
1
+ import * as fs from "node:fs";
2
+ import { type Either } from "purify-ts";
3
+ export declare function stat(path: fs.PathLike): Promise<Either<Error, fs.Stats>>;
4
+ //# sourceMappingURL=stat.d.ts.map
package/dist/stat.js ADDED
@@ -0,0 +1,11 @@
1
+ import * as fs from "node:fs";
2
+ import { Left, Right } from "purify-ts";
3
+ export async function stat(path) {
4
+ try {
5
+ return Right(await fs.promises.stat(path));
6
+ }
7
+ catch (e) {
8
+ return Left(e);
9
+ }
10
+ }
11
+ //# sourceMappingURL=stat.js.map
package/package.json ADDED
@@ -0,0 +1,39 @@
1
+ {
2
+ "dependencies": {
3
+ "@rdfjs/formats": "~4.0.1",
4
+ "@rdfjs/types": "~2.0.1",
5
+ "@types/rdfjs__formats": "~4.0.1",
6
+ "@types/unbzip2-stream": "~1.4.3",
7
+ "mime": "~4.1.0",
8
+ "purify-ts": "~2.1.4",
9
+ "ts-log": "~3.0.2",
10
+ "unbzip2-stream": "~1.4.3"
11
+ },
12
+ "devDependencies": {
13
+ "@rdfjs/dataset": "~2.0.2",
14
+ "@types/rdfjs__dataset": "~2.0.7"
15
+ },
16
+ "files": [
17
+ "dist/*.d.ts",
18
+ "dist/*.js"
19
+ ],
20
+ "license": "Apache-2.0",
21
+ "main": "./dist/index.js",
22
+ "name": "@rdfx/fs",
23
+ "packageManager": "npm@11.11.0",
24
+ "repository": {
25
+ "type": "git",
26
+ "url": "git+https://github.com/minorg/rdfx.git"
27
+ },
28
+ "scripts": {
29
+ "build": "tsc -b",
30
+ "clean": "rimraf dist",
31
+ "depcheck": "depcheck .",
32
+ "dev": "tsc -w --preserveWatchOutput",
33
+ "dev:tests": "tsc -p __tests__ -w --preserveWatchOutput",
34
+ "test": "cd ../.. && vitest run --project @rdfx/fs"
35
+ },
36
+ "type": "module",
37
+ "types": "./dist/index.d.ts",
38
+ "version": "0.0.2"
39
+ }