@mdgf11/filesystem-lib 2.0.0

package/dist/filesystemDocumentMethods.d.ts

@@ -0,0 +1,7 @@
+import { PartialJurisprudenciaDocument } from "@stjiris/jurisprudencia-document";
+import { ContentType, Date_Area_Section, FilesystemDocument, Retrievable_Metadata, Sharepoint_Metadata } from "./types";
+export declare function writeFilesystemDocument(filesystem_document: FilesystemDocument): void;
+export declare function loadFilesystemDocument(jsonPath: string): FilesystemDocument;
+export declare function createJurisprudenciaDocument(retrievable_Metadata: Retrievable_Metadata, contents: ContentType[], date_area_section: Date_Area_Section, sharepoint_metadata?: Sharepoint_Metadata): Promise<PartialJurisprudenciaDocument>;
+export declare function hasSelectableText(buffer: Buffer): Promise<boolean>;
+export declare function generateFilePath(date_area_section: Date_Area_Section, retrievable_metadata: Retrievable_Metadata): string;

package/dist/filesystemDocumentMethods.js

@@ -0,0 +1,227 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeFilesystemDocument = writeFilesystemDocument;
+exports.loadFilesystemDocument = loadFilesystemDocument;
+exports.createJurisprudenciaDocument = createJurisprudenciaDocument;
+exports.hasSelectableText = hasSelectableText;
+exports.generateFilePath = generateFilePath;
+const jurisprudencia_document_1 = require("@stjiris/jurisprudencia-document");
+const fs_1 = __importDefault(require("fs"));
+const mammoth_1 = __importDefault(require("mammoth"));
+const pdfjsLib = __importStar(require("pdfjs-dist/legacy/build/pdf.mjs"));
+const types_1 = require("./types");
+const descritores_1 = require("./descritores");
+function writeFilesystemDocument(filesystem_document) {
+    if (!filesystem_document.content)
+        return;
+    const safe = {
+        ...filesystem_document,
+        content: filesystem_document.content?.map(({ extension }) => ({ extension }))
+    };
+    const content = filesystem_document.content;
+    if (filesystem_document.file_path) {
+        // make filesystem paths
+        const filesystem_dir_path = `${types_1.ROOT_PATH}${types_1.FILESYSTEM_PATH}${filesystem_document.file_path}`;
+        const filesystem_metadata_path = `${filesystem_dir_path}/${types_1.DETAILS_NAME}.json`;
+        fs_1.default.mkdirSync(filesystem_dir_path, { recursive: true });
+        fs_1.default.writeFileSync(filesystem_metadata_path, JSON.stringify(safe, null, 2), { encoding: "utf-8" });
+        for (const content_i of content) {
+            const filesystem_original_path = `${filesystem_dir_path}/${types_1.ORIGINAL_NAME}.${content_i.extension}`;
+            fs_1.default.writeFileSync(filesystem_original_path, content_i.data, { encoding: "utf-8" });
+        }
+        // make metadata copy on filesystem copy
+        if (filesystem_document.sharepoint_metadata) {
+            const filesystem_sharepoint_dir_path = `${types_1.ROOT_PATH}${types_1.SHAREPOINT_COPY_PATH}${filesystem_document.sharepoint_metadata.sharepoint_path_rel}`;
+            const filesystem_sharepoint_path = `${filesystem_sharepoint_dir_path}/${types_1.DETAILS_NAME}.json`;
+            fs_1.default.mkdirSync(filesystem_sharepoint_dir_path, { recursive: true });
+            fs_1.default.writeFileSync(filesystem_sharepoint_path, JSON.stringify(safe, null, 2), { encoding: "utf-8" });
+        }
+    }
+    else {
+        if (filesystem_document.sharepoint_metadata) {
+            const filesystem_sharepoint_dir_path = `${types_1.ROOT_PATH}${types_1.SHAREPOINT_COPY_PATH}${filesystem_document.sharepoint_metadata.sharepoint_path_rel}`;
+            const filesystem_sharepoint_path = `${filesystem_sharepoint_dir_path}/${types_1.DETAILS_NAME}.json`;
+            fs_1.default.mkdirSync(filesystem_sharepoint_dir_path, { recursive: true });
+            fs_1.default.writeFileSync(filesystem_sharepoint_path, JSON.stringify(safe, null, 2), { encoding: "utf-8" });
+            for (const content_i of content) {
+                const filesystem_original_path = `${filesystem_sharepoint_dir_path}/${types_1.ORIGINAL_NAME}.${content_i.extension}`;
+                fs_1.default.writeFileSync(filesystem_original_path, content_i.data, { encoding: "utf-8" });
+            }
+        }
+    }
+}
+function loadFilesystemDocument(jsonPath) {
+    const jsonString = fs_1.default.readFileSync(jsonPath, 'utf-8');
+    const parsed = JSON.parse(jsonString);
+    return {
+        ...parsed,
+        creation_date: new Date(parsed.creation_date),
+        last_update_date: new Date(parsed.last_update_date),
+        content: parsed.content?.map((item) => ({
+            extension: item.extension,
+            data: Buffer.from([])
+        }))
+    };
+}
+async function createJurisprudenciaDocument(retrievable_Metadata, contents, date_area_section, sharepoint_metadata) {
+    if (!retrievable_Metadata) {
+        throw new Error("Missing metadata.");
+    }
+    const content = await extractContent(contents);
+    const url = sharepoint_metadata ? sharepoint_metadata.sharepoint_url : "";
+    let Original = {};
+    let CONTENT = content;
+    let numProc = retrievable_Metadata.process_number;
+    let Data = Intl.DateTimeFormat("pt-PT").format(date_area_section.file_date);
+    let origin = "STJ (Sharepoint)";
+    Original["Decisão Texto Integral"] = content.map(line => `<p><font>${line}</font><br>`).join('');
+    Original["Data"] = Data;
+    Original["Número de Processo"] = numProc;
+    Original["Fonte"] = origin;
+    Original["URL"] = url;
+    Original["Jurisprudência"] = "Simples";
+    let obj = {
+        "Original": Original,
+        "CONTENT": CONTENT,
+        "Data": Data,
+        "Número de Processo": numProc,
+        "Fonte": origin,
+        "URL": url,
+        "Jurisprudência": { Index: ["Simples"], Original: ["Simples"], Show: ["Simples"] },
+        "STATE": "importação",
+    };
+    obj.Sumário = "";
+    obj.Texto = content.map(line => `<p><font>${line}</font><br>`).join('');
+    if (retrievable_Metadata.descriptors && retrievable_Metadata.descriptors.length > 0) {
+        obj.Descritores = {
+            Index: retrievable_Metadata.descriptors.map(desc => descritores_1.DescritorOficial[desc]),
+            Original: retrievable_Metadata.descriptors,
+            Show: retrievable_Metadata.descriptors.map(desc => descritores_1.DescritorOficial[desc])
+        };
+    }
+    if (date_area_section.area && date_area_section.area.length > 0) {
+        obj.Área = { Index: [date_area_section.area], Original: [date_area_section.area], Show: [date_area_section.area] };
+    }
+    if (date_area_section.section && date_area_section.section.length > 0) {
+        obj.Secção = { Index: [date_area_section.section], Original: [date_area_section.section], Show: [date_area_section.section] };
+    }
+    if (retrievable_Metadata.judge && retrievable_Metadata.judge.length > 0) {
+        obj["Relator Nome Profissional"] = { Index: [retrievable_Metadata.judge], Original: [retrievable_Metadata.judge], Show: [retrievable_Metadata.judge] };
+    }
+    if (retrievable_Metadata.process_mean && retrievable_Metadata.process_mean.length > 0) {
+        obj["Meio Processual"] = { Index: [retrievable_Metadata.process_mean], Original: [retrievable_Metadata.process_mean], Show: [retrievable_Metadata.process_mean] };
+    }
+    if (retrievable_Metadata.decision && retrievable_Metadata.decision.length > 0) {
+        obj["Decisão"] = { Index: [retrievable_Metadata.decision], Original: [retrievable_Metadata.decision], Show: [retrievable_Metadata.decision] };
+    }
+    obj["HASH"] = (0, jurisprudencia_document_1.calculateHASH)({
+        ...obj,
+        Original: obj.Original,
+        "Número de Processo": obj["Número de Processo"] || "",
+        Sumário: obj.Sumário || "",
+        Texto: obj.Texto || "",
+    });
+    obj["UUID"] = (0, jurisprudencia_document_1.calculateUUID)(obj["HASH"]);
+    return obj;
+}
+async function hasSelectableText(buffer) {
+    try {
+        const uint8Array = new Uint8Array(buffer);
+        const loadingTask = pdfjsLib.getDocument({
+            data: uint8Array,
+            standardFontDataUrl: 'node_modules/pdfjs-dist/standard_fonts/',
+        });
+        const pdf = await loadingTask.promise;
+        const pagesToCheck = Math.min(3, pdf.numPages);
+        for (let i = 1; i <= pagesToCheck; i++) {
+            const page = await pdf.getPage(i);
+            const textContent = await page.getTextContent();
+            if (textContent.items.some((item) => item.str?.trim().length > 0)) {
+                return true;
+            }
+        }
+        return false;
+    }
+    catch (error) {
+        console.error('Error reading PDF:', error);
+        return false;
+    }
+}
+function generateFilePath(date_area_section, retrievable_metadata) {
+    return `/${date_area_section.area}/${date_area_section.file_date.getFullYear()}/${date_area_section.file_date.getMonth() + 1}/${date_area_section.file_date.getDate()}/${retrievable_metadata.process_number.replace("/", "-")}`;
+}
+async function extractContent(contents) {
+    for (const content of contents) {
+        if (content.extension === "txt") {
+            return content.data.toString('utf-8').split(/\r?\n/).filter(line => line.trim().length > 0);
+        }
+        if (content.extension === "pdf") {
+            return await pdfToLines(content.data);
+        }
+        if (content.extension === "docx") {
+            return await docxToLines(content.data);
+        }
+    }
+    throw new Error("Contents are not a supported format.");
+}
+async function pdfToLines(buffer) {
+    const uint8Array = new Uint8Array(buffer);
+    const loadingTask = pdfjsLib.getDocument({ data: uint8Array, verbosity: 0 });
+    const pdf = await loadingTask.promise;
+    const allLines = [];
+    for (let i = 1; i <= pdf.numPages; i++) {
+        const page = await pdf.getPage(i);
+        const textContent = await page.getTextContent();
+        const pageText = textContent.items
+            .map((item) => item.str)
+            .join('\n');
+        const lines = pageText.split('\n').filter(line => line.trim().length > 0);
+        allLines.push(...lines);
+    }
+    return allLines;
+}
+async function docxToLines(buffer) {
+    const result = await mammoth_1.default.extractRawText({ buffer });
+    const text = result.value || "";
+    const content = text
+        .split(/\r?\n/)
+        .map(line => line.trim())
+        .filter(Boolean);
+    return content;
+}

package/dist/filesystemUpdateMethods.d.ts

@@ -0,0 +1,5 @@
+import { FilesystemDocument, FilesystemUpdate } from "./types";
+export declare function addFileToUpdate(update: FilesystemUpdate, filesystem_document: FilesystemDocument): void;
+export declare function writeFilesystemUpdate(update: FilesystemUpdate): void;
+export declare function logDocumentProcessingError(update: FilesystemUpdate, err: string): void;
+export declare function loadLastFilesystemUpdate(): FilesystemUpdate;

package/dist/filesystemUpdateMethods.js

@@ -0,0 +1,74 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.addFileToUpdate = addFileToUpdate;
+exports.writeFilesystemUpdate = writeFilesystemUpdate;
+exports.logDocumentProcessingError = logDocumentProcessingError;
+exports.loadLastFilesystemUpdate = loadLastFilesystemUpdate;
+const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const types_1 = require("./types");
+function addFileToUpdate(update, filesystem_document) {
+    if (!filesystem_document.file_path) {
+        throw new Error("File to be added to update doesn't have a system path.");
+    }
+    if (!update.created) {
+        update.created = [];
+    }
+    if (!update.created_num) {
+        update.created_num = 0;
+    }
+    update.created_num += 1;
+    update.created.push(filesystem_document.file_path);
+}
+function writeFilesystemUpdate(update) {
+    update.date_end = new Date();
+    fs_1.default.mkdirSync(types_1.UPDATE_DIR, { recursive: true });
+    const updates_file_path = `${types_1.UPDATE_DIR}/log_${formatUpdateDate(update.date_end)}.json`;
+    const drive_dir_path = `${types_1.UPDATE_DIR}/All`;
+    fs_1.default.mkdirSync(drive_dir_path, { recursive: true });
+    const drive_file_path = `${drive_dir_path}/log_${formatUpdateDate(update.date_end)}.json`;
+    removeOldUpdate(types_1.UPDATE_DIR);
+    fs_1.default.writeFileSync(drive_file_path, JSON.stringify(update, null, 2), { encoding: "utf-8" });
+    fs_1.default.writeFileSync(updates_file_path, JSON.stringify(update, null, 2), { encoding: "utf-8" });
+}
+function logDocumentProcessingError(update, err) {
+    update.file_errors.push(err);
+}
+function loadLastFilesystemUpdate() {
+    const empty_update = {
+        updateSource: "STJ (Sharepoint)",
+        file_errors: [],
+        date_start: new Date()
+    };
+    if (!fs_1.default.existsSync(types_1.UPDATE_DIR))
+        return empty_update;
+    const files = fs_1.default.readdirSync(types_1.UPDATE_DIR);
+    for (const file of files) {
+        const fullPath = path_1.default.join(types_1.UPDATE_DIR, file);
+        if (fs_1.default.statSync(fullPath).isFile() && file.toLowerCase().includes("log")) {
+            const jsonString = fs_1.default.readFileSync(fullPath, 'utf-8');
+            const parsed = JSON.parse(jsonString);
+            return parsed;
+        }
+    }
+    return empty_update;
+}
+function formatUpdateDate(d = new Date()) {
+    const pad = (n) => n.toString().padStart(2, "0");
+    return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}_${pad(d.getHours())}-${pad(d.getMinutes())}-${pad(d.getSeconds())}`;
+}
+function removeOldUpdate(folderPath) {
+    if (!fs_1.default.existsSync(folderPath))
+        return;
+    const files = fs_1.default.readdirSync(folderPath);
+    for (const file of files) {
+        const fullPath = path_1.default.join(folderPath, file);
+        if (fs_1.default.statSync(fullPath).isFile() && file.toLowerCase().includes("log")) {
+            fs_1.default.unlinkSync(fullPath);
+            console.log(`Deleted: ${fullPath} `);
+        }
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export * from "./filesystemDocumentMethods";
+export * from "./filesystemUpdateMethods";
+export * from "./types";

package/dist/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./filesystemDocumentMethods"), exports);
+__exportStar(require("./filesystemUpdateMethods"), exports);
+__exportStar(require("./types"), exports);

package/dist/types.d.ts

@@ -0,0 +1,62 @@
+export declare const UpdateSources: readonly ["STJ (Sharepoint)", "Juris"];
+export type SupportedUpdateSources = typeof UpdateSources[number];
+import { PartialJurisprudenciaDocument } from '@stjiris/jurisprudencia-document';
+export declare const ROOT_PATH: string;
+export declare const FILESYSTEM_PATH = "/FileSystem";
+export declare const SHAREPOINT_COPY_PATH = "/Sharepoint";
+export declare const DETAILS_NAME = "Detalhes";
+export declare const ORIGINAL_NAME = "Original";
+export declare const LOGS_PATH = "/Updates";
+export declare const UPDATE_DIR: string;
+export type FilesystemUpdate = {
+    updateSource: SupportedUpdateSources;
+    date_start: Date;
+    file_errors: string[];
+    date_end?: Date;
+    created_num?: number;
+    created?: string[];
+    deleted_num?: number;
+    deleted?: string[];
+    updated_num?: number;
+    updated?: string[];
+    next_link?: string;
+    delta_link?: string;
+};
+export type Sharepoint_Metadata = {
+    drive_name: string;
+    drive_id: string;
+    sharepoint_id: string;
+    parent_sharepoint_id: string;
+    sharepoint_path: string;
+    sharepoint_path_rel: string;
+    sharepoint_url: string;
+    extensions: Supported_Content_Extensions[];
+    xor_hash?: string;
+};
+export type Retrievable_Metadata = {
+    process_number: string;
+    judge: string;
+    process_mean: string;
+    decision: string;
+    descriptors?: string[];
+};
+export type Date_Area_Section = {
+    file_date: Date;
+    area: string;
+    section: string;
+};
+export declare const SUPPORTED_EXTENSIONS: readonly ["txt", "pdf", "docx"];
+export type Supported_Content_Extensions = typeof SUPPORTED_EXTENSIONS[number];
+export type ContentType = {
+    extension: Supported_Content_Extensions;
+    data: Buffer;
+};
+export type FilesystemDocument = {
+    creation_date: Date;
+    last_update_date: Date;
+    jurisprudencia_document: PartialJurisprudenciaDocument;
+    file_path: string;
+    sharepoint_metadata?: Sharepoint_Metadata;
+    content?: ContentType[];
+};
+export declare function isSupportedExtension(ext: string): ext is Supported_Content_Extensions;

package/dist/types.js

@@ -0,0 +1,21 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SUPPORTED_EXTENSIONS = exports.UPDATE_DIR = exports.LOGS_PATH = exports.ORIGINAL_NAME = exports.DETAILS_NAME = exports.SHAREPOINT_COPY_PATH = exports.FILESYSTEM_PATH = exports.ROOT_PATH = exports.UpdateSources = void 0;
+exports.isSupportedExtension = isSupportedExtension;
+exports.UpdateSources = ["STJ (Sharepoint)", "Juris"];
+const dotenv_1 = __importDefault(require("dotenv"));
+dotenv_1.default.config();
+exports.ROOT_PATH = process.env['LOCAL_ROOT'] || 'results';
+exports.FILESYSTEM_PATH = `/FileSystem`;
+exports.SHAREPOINT_COPY_PATH = `/Sharepoint`;
+exports.DETAILS_NAME = "Detalhes";
+exports.ORIGINAL_NAME = "Original";
+exports.LOGS_PATH = "/Updates";
+exports.UPDATE_DIR = `${exports.ROOT_PATH}${exports.LOGS_PATH}`;
+exports.SUPPORTED_EXTENSIONS = ["txt", "pdf", "docx"];
+function isSupportedExtension(ext) {
+    return exports.SUPPORTED_EXTENSIONS.includes(ext);
+}

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@mdgf11/filesystem-lib",
+  "version": "2.0.0",
+  "description": "Library to extend usage of jurisprudencia-document",
+  "files": [
+    "dist"
+  ],
+  "license": "ISC",
+  "author": "Miguel Fonseca",
+  "type": "commonjs",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "run": "node dist/index.js",
+    "start": "npm run build && npm run run"
+  },
+  "dependencies": {
+    "@stjiris/jurisprudencia-document": "npm:@mdgf11/jurisprudencia-document@^13.1.0",
+    "dotenv": "^17.2.2",
+    "mammoth": "^1.10.0",
+    "pdfjs-dist": "^5.4.530"
+  },
+  "devDependencies": {
+    "@types/node": "^20.19.24",
+    "ts-jest": "^29.1.1",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.9.3"
+  }
+}