@tiddlywiki/mws 0.0.12 → 0.0.13

package/src/routes/services/attachments.ts

@@ -1,259 +0,0 @@
-import { tryParseJSON } from "../../utils";
-import * as path from "path";
-import * as fs from "fs";
-import { SiteConfig } from "../router";
-import sjcl from "sjcl";
-export interface TiddlerFields extends Record<string, any> {
-  title: PrismaField<"Tiddlers", "title">;
-}
-const $tw: any = {};
-
-
-/*
-
-Class to handle the attachments in the filing system
-
-The store folder looks like this:
-
-store/
-	inbox/ - files that are in the process of being uploaded via a multipart form upload
-		202402282125432742/
-			0
-			1
-			...
-		...
-	files/ - files that are the text content of large tiddlers
-		b7def178-79c4-4d88-b7a4-39763014a58b/
-			data.jpg - the extension is provided for convenience when directly inspecting the file system
-			meta.json - contains:
-				{
-					"filename": "data.jpg",
-					"type": "video/mp4",
-					"uploaded": "2024021821224823"
-				}
-	database.sql - The database file (managed by sql-tiddler-database.js)
-
-*/
-
-export class AttachmentService {
-
-  constructor(
-    protected config: SiteConfig,
-    protected prisma: PrismaTxnClient,
-  ) { 
-
-  }
-
-  makeCanonicalUri(bag_name: string, title: string) {
-    return `/bags/${encodeURIComponentExtended(bag_name)}/tiddlers/${encodeURIComponentExtended(title)}/blob`;
-  }
-  /*
-  Given tiddler fields, tiddler_id and a bag_name, return the tiddler fields after the following process:
-  - Apply the tiddler_id as the revision field
-  - Apply the bag_name as the bag field
-  */
-  processOutgoingTiddler({ tiddler, tiddler_id, bag_name, attachment_hash }: {
-    tiddler: TiddlerFields;
-    tiddler_id: any;
-    bag_name: PrismaField<"Bags", "bag_name">;
-    attachment_hash: PrismaField<"Tiddlers", "attachment_hash">;
-  }) {
-    if (attachment_hash !== null) {
-      return Object.assign(
-        {},
-        tiddler,
-        {
-          text: undefined,
-          _canonical_uri: this.makeCanonicalUri(bag_name, tiddler.title),
-          revision: tiddler_id,
-          bag: bag_name
-        }
-      );
-    } else {
-      return tiddler;
-    }
-  }
-  /*
-  */
-  async processIncomingTiddler({
-    tiddlerFields, existing_attachment_hash, existing_canonical_uri
-  }: {
-    tiddlerFields: TiddlerFields;
-    existing_attachment_hash: PrismaField<"Tiddlers", "attachment_hash">;
-    existing_canonical_uri: any;
-  }) {
-    const { attachmentSizeLimit, attachmentsEnabled, } = this.config;
-    const contentTypeInfo = this.config.contentTypeInfo[tiddlerFields.type || "text/vnd.tiddlywiki"];
-    const isBinary = !!contentTypeInfo && contentTypeInfo.encoding === "base64";
-
-    let shouldProcessAttachment = tiddlerFields.text && tiddlerFields.text.length > attachmentSizeLimit;
-
-    if (existing_attachment_hash) {
-      const fileSize = await this.getAttachmentFileSize(existing_attachment_hash);
-      if (fileSize && fileSize <= attachmentSizeLimit) {
-        const existingAttachmentMeta = await this.getAttachmentMetadata(existing_attachment_hash);
-        const hasCanonicalField = !!tiddlerFields._canonical_uri;
-        const skipAttachment = hasCanonicalField && (tiddlerFields._canonical_uri === (existingAttachmentMeta ? existingAttachmentMeta._canonical_uri : existing_canonical_uri));
-        shouldProcessAttachment = !skipAttachment;
-      } else {
-        shouldProcessAttachment = false;
-      }
-    }
-
-    if (attachmentsEnabled && isBinary && shouldProcessAttachment) {
-      const attachment_hash = existing_attachment_hash || await this.saveAttachment({
-        text: tiddlerFields.text,
-        type: tiddlerFields.type,
-        reference: tiddlerFields.title,
-        _canonical_uri: tiddlerFields._canonical_uri
-      });
-
-      if (tiddlerFields && tiddlerFields._canonical_uri) {
-        delete tiddlerFields._canonical_uri;
-      }
-
-      return {
-        tiddlerFields: Object.assign({}, tiddlerFields, { text: undefined }),
-        attachment_hash: attachment_hash
-      };
-    } else {
-      return {
-        tiddlerFields: tiddlerFields,
-        attachment_hash: existing_attachment_hash
-      };
-    }
-  }
-
-  /*
-  Check if an attachment name is valid (is a 64 character hex string)
-  */
-  isValidAttachmentName(content_hash: string) {
-    const re = new RegExp('^[a-f0-9]{64}$');
-    return re.test(content_hash);
-  }
-  /*
-  Saves an attachment to a file. Options include:
-  
-  text: text content (may be binary)
-  type: MIME type of content
-  reference: reference to use for debugging
-  _canonical_uri: canonical uri of the content
-  */
-  async saveAttachment(options: {
-    text: string;
-    type: string;
-    reference: string;
-    _canonical_uri: string;
-  }) {
-
-    // Compute the content hash for naming the attachment
-    const contentHash = sjcl.codec.hex.fromBits($tw.sjcl.hash.sha256.hash(options.text)).slice(0, 64).toString();
-    // Choose the best file extension for the attachment given its type
-    const contentTypeInfo = this.config.contentTypeInfo[options.type] || this.config.contentTypeInfo["application/octet-stream"];
-    // Creat the attachment directory
-    const attachmentPath = path.resolve(this.config.storePath, "files", contentHash);
-    createDirectory(attachmentPath);
-    // Save the data file
-    const dataFilename = "data" + contentTypeInfo.extension;
-    fs.writeFileSync(path.resolve(attachmentPath, dataFilename), options.text, contentTypeInfo.encoding);
-    // Save the meta.json file
-    this.writeMetaFile(fs, path, attachmentPath, options, contentHash, dataFilename);
-    return contentHash as PrismaField<"Tiddlers", "attachment_hash">;
-  }
-  private writeMetaFile(fs: any, path: any, attachmentPath: any, options: { text: string; type: string; reference: string; _canonical_uri: string; }, contentHash: any, dataFilename: string) {
-    fs.writeFileSync(path.resolve(attachmentPath, "meta.json"), JSON.stringify({
-      _canonical_uri: options._canonical_uri,
-      created: new Date().toISOString(),
-      modified: new Date().toISOString(),
-      contentHash: contentHash,
-      filename: dataFilename,
-      type: options.type
-    }, null, 4));
-  }
-
-  /*
-  Adopts an attachment file into the store (e.g. move from a temp upload directory)
-  */
-  async adoptAttachment({ incomingFilepath, type, hash, _canonical_uri }: {
-    incomingFilepath: any;
-    type: string | number;
-    hash: any;
-    _canonical_uri: any;
-  }) {
-    // Choose the best file extension for the attachment given its type
-    const contentTypeInfo = this.config.contentTypeInfo[type] || this.config.contentTypeInfo["application/octet-stream"];
-    // Creat the attachment directory
-    const attachmentPath = path.resolve(this.config.storePath, "files", hash);
-    createDirectory(attachmentPath);
-    // Rename the data file
-    const dataFilename = "data" + contentTypeInfo.extension, dataFilepath = path.resolve(attachmentPath, dataFilename);
-    fs.renameSync(incomingFilepath, dataFilepath);
-    // Save the meta.json file
-    fs.writeFileSync(path.resolve(attachmentPath, "meta.json"), JSON.stringify({
-      _canonical_uri: _canonical_uri,
-      created: new Date().toISOString(),
-      modified: new Date().toISOString(),
-      contentHash: hash,
-      filename: dataFilename,
-      type: type
-    }, null, 4));
-    return hash;
-  }
-  /*
-  Get an attachment ready to stream. Returns null if there is an error or:
-  stream: filestream of file
-  type: type of file
-  */
-  async getAttachmentStream(content_hash: string) {
-    // Check the attachment name
-    if (!this.isValidAttachmentName(content_hash)) return null;
-
-    // Construct the path to the attachment directory
-    const attachmentPath = path.resolve(this.config.storePath, "files", content_hash);
-
-    // Read the meta.json file
-    const metaJsonPath = path.resolve(attachmentPath, "meta.json");
-    if (!fs.existsSync(metaJsonPath) || !fs.statSync(metaJsonPath).isFile()) return null;
-
-    const meta = tryParseJSON<any>(fs.readFileSync(metaJsonPath, "utf8"));
-    if (!meta) return null;
-
-    const dataFilepath = path.resolve(attachmentPath, meta.filename);
-    // Check if the data file exists
-    if (!fs.existsSync(dataFilepath) || !fs.statSync(dataFilepath).isFile()) return null
-
-    // Stream the file
-    return {
-      stream: fs.createReadStream(dataFilepath),
-      type: meta.type
-    };
-
-
-  }
-  /*
-  Get the size of an attachment file given the contentHash.
-  Returns the size in bytes, or null if the file doesn't exist.
-  */
-  async getAttachmentFileSize(content_hash: PrismaField<"Tiddlers", "attachment_hash"> & {}) {
-    // Construct the path to the attachment directory
-    const attachmentPath = path.resolve(this.config.storePath, "files", content_hash);
-    // Read the meta.json file
-    const metaJsonPath = path.resolve(attachmentPath, "meta.json");
-    if (!(fs.existsSync(metaJsonPath) && fs.statSync(metaJsonPath).isFile())) return null
-    const meta = tryParseJSON<any>(fs.readFileSync(metaJsonPath, "utf8"));
-    if (!meta) return null;
-    const dataFilepath = path.resolve(attachmentPath, meta.filename);
-    // Check if the data file exists and return its size
-    if (!(fs.existsSync(dataFilepath) && fs.statSync(dataFilepath).isFile())) return null;
-    return fs.statSync(dataFilepath).size;
-  }
-
-  async getAttachmentMetadata(content_hash: PrismaField<"Tiddlers", "attachment_hash"> & {}) {
-    const attachmentPath = path.resolve(this.config.storePath, "files", content_hash);
-    const metaJsonPath = path.resolve(attachmentPath, "meta.json");
-    if (!fs.existsSync(metaJsonPath)) return null;
-    return JSON.parse(fs.readFileSync(metaJsonPath, "utf8"));
-  }
-
-}
-

package/src/routes/services/sessions.ts

@@ -1,166 +0,0 @@
-
-import { Streamer } from "../../streamer";
-import { randomBytes } from "node:crypto";
-import { registerZodRoutes, Router, RouterKeyMap, RouterRouteMap, zodRoute, ZodState } from "../router";
-import { StateObject } from "../../StateObject";
-import { z } from "zod";
-import { JsonValue, Z2 } from "../../utils";
-
-export interface AuthUser {
-  /** User ID. 0 if the user is not logged in. */
-  user_id: PrismaField<"Users", "user_id">;
-  /** User role_ids. This may have length even if the user isn't logged in, to allow ACL for anon. */
-  role_ids: PrismaField<"Roles", "role_id">[];
-  /** Username passed to the client */
-  username: PrismaField<"Users", "username">;
-  /** A session_id isn't guarenteed. There may be a session even if the user isn't logged in, and may not be even if they are, depending on the the situation. */
-  sessionId: PrismaField<"Sessions", "session_id"> | undefined;
-  /** Is this user considered a site-admin. This is determined by the auth service, not MWS. */
-  isAdmin: boolean;
-  /** Is the user logged in? This also means that user_id should be 0. role_ids may still be specified. */
-  isLoggedIn: boolean;
-}
-
-export const SessionKeyMap: RouterKeyMap<SessionManager, true> = {
-  login1: true,
-  login2: true,
-  logout: true,
-}
-
-
-/**
- * 
- * @param path path starting with a forward slash
- * @param zodRequest the zod for state.data
- * @param inner the handler to call
- * @returns the ZodRoute
- */
-export function zodSession<T extends z.ZodTypeAny, R extends JsonValue>(
-  path: string,
-  zodRequest: (z: Z2<"JSON">) => T,
-  inner: (state: ZodState<"POST", "json", Record<string, z.ZodTypeAny>, T>, prisma: PrismaTxnClient) => Promise<R>
-) {
-  return zodRoute(["POST"], path, z => ({}), "json", zodRequest, async state => {
-    return state.$transaction(async (prisma) => await inner(state, prisma));
-  });
-}
-
-
-export type SessionManagerMap = RouterRouteMap<SessionManager>;
-
-export class SessionManager {
-
-  static async defineRoutes(root: rootRoute) {
-    registerZodRoutes(root, new SessionManager(), Object.keys(SessionKeyMap))
-  }
-
-  static async parseIncomingRequest(streamer: Streamer, router: Router): Promise<AuthUser> {
-
-    const sessionId = streamer.cookies.session as PrismaField<"Sessions", "session_id"> | undefined;
-    const session = sessionId && await router.engine.sessions.findUnique({
-      where: { session_id: sessionId },
-      select: { user: { select: { user_id: true, username: true, roles: { select: { role_id: true } } } } }
-    });
-
-    if (sessionId && session) return {
-      user_id: session.user.user_id,
-      username: session.user.username,
-      isAdmin: session.user.roles.some(e => e.role_id === 1),
-      role_ids: session.user.roles.map(e => e.role_id),
-      sessionId,
-      isLoggedIn: true,
-    };
-    else return {
-      user_id: 0 as PrismaField<"Users", "user_id">,
-      username: "(anon)" as PrismaField<"Users", "username">,
-      isAdmin: false,
-      role_ids: [] as PrismaField<"Roles", "role_id">[],
-      sessionId: undefined as PrismaField<"Sessions", "session_id"> | undefined,
-      isLoggedIn: false,
-    };
-  }
-
-  login1 = zodSession("/login/1", z => z.object({
-    username: z.prismaField("Users", "username", "string"),
-    startLoginRequest: z.string(),
-  }), async (state, prisma) => {
-    const { username, startLoginRequest } = state.data;
-
-    const user = await prisma.users.findUnique({
-      where: { username },
-      select: { user_id: true, password: true, }
-    });
-
-    if (!user) throw "User not found.";
-
-    const { user_id, password } = user;
-
-    const stater = state.PasswordService.LoginGenerator({
-      user_id,
-      startLoginRequest,
-      registrationRecord: password,
-    });
-
-    const loginResponse = await stater.next(0);
-
-    if (loginResponse.done) throw "Login failed.";
-
-    const loginSession = await state.PasswordService.startLoginSession(stater);
-
-    state.setCookie("loginsession", loginSession, { httpOnly: true, path: "/", secure: state.isSecure, sameSite: "Strict" });
-
-    return { loginResponse: loginResponse.value };
-
-  })
-
-  login2 = zodSession("/login/2", z => z.object({
-    finishLoginRequest: z.string()
-  }), async (state, prisma) => {
-    const { finishLoginRequest } = state.data;
-
-    if (!state.cookies.loginsession) throw "Login session not found.";
-
-    const stater = state.PasswordService.serverState.get(state.cookies.loginsession);
-
-    if (!stater) throw "Login session not found.";
-
-    const { done, value } = await stater.next(1, finishLoginRequest);
-
-    if (!value?.session?.sessionKey) throw "Login failed.";
-
-    const session_id = await this.createSession(prisma, value.user_id, value.session.sessionKey);
-
-    state.setCookie("session", session_id, { httpOnly: true, path: "/", secure: state.isSecure, sameSite: "Strict" });
-
-    return null;
-
-  })
-
-  logout = zodSession("/logout", z => z.undefined(), async (state, prisma) => {
-
-    if (state.user.isLoggedIn) {
-      await prisma.sessions.delete({ where: { session_id: state.user.sessionId } });
-    }
-    var cookies = state.headers.cookie ? state.headers.cookie.split(";") : [];
-    for (var i = 0; i < cookies.length; i++) {
-      var cookie = cookies[i]?.trim().split("=")[0];
-      if (!cookie) continue;
-      // state.setHeader("Set-Cookie", cookie + "=; HttpOnly; Path=/; Expires=Thu, 01 Jan 1970 00:00:00 GMT; SameSite=Strict");
-      state.setCookie(cookie, "", { httpOnly: true, path: "/", expires: new Date(0), secure: state.isSecure, sameSite: "Strict" });
-    }
-
-    return null;
-  });
-
-  async createSession(prisma: PrismaTxnClient, user_id: PrismaField<"Users", "user_id">, session_key: string) {
-    const session_id = randomBytes(16).toString("base64url");
-    return await prisma.sessions.create({
-      data: {
-        user_id,
-        session_key,
-        session_id,
-        last_accessed: new Date(),
-      }
-    }).then(({ session_id }) => session_id);
-  }
-}

package/src/routes/tw-test.ts

@@ -1,65 +0,0 @@
-import { Server } from "http";
-import { rootRoute } from "./router";
-import { ZodAssert as zodAssert } from "../utils";
-import { Tiddler, TiddlyWiki, TW } from "tiddlywiki";
-
-export function TWRoutes(parent: rootRoute) {
-  const wikiRouter = new TWRouter();
-  parent.defineRoute({
-    method: ["OPTIONS", "GET", "HEAD", "POST", "PUT"],
-    bodyFormat: "stream",
-    path: /^\/mws(\/|$)/,
-    pathParams: ["final_slash"]
-  }, async (state) => {
-    zodAssert.pathParams(state, z => ({ final_slash: z.string().optional() }));
-    if (!state.pathParams.final_slash) throw state.redirect(state.urlInfo.pathname + "/");
-    // literally don't do this
-    const { req, res } = (state as any).streamer;
-    // options not required
-    wikiRouter.twserver.requestHandler(req, res, {} as any);
-  });
-}
-
-class TWRouter {
-  $tw: TW;
-  twserver!: any;
-
-  constructor() {
-    this.$tw = TiddlyWiki();
-    // tiddlywiki [+<pluginname> | ++<pluginpath>] [<wikipath>] ...[--command ...args]
-    this.$tw.boot.argv = [
-      // "++plugins/client",
-      // "++plugins/server",
-      // "+tiddlywiki/filesystem",
-      // "+tiddlywiki/tiddlyweb",
-      // "./editions/mws-server",
-      // "--mws-load-plugin-bags",
-      // "--build", "load-mws-demo-data",
-      // "--mws-listen", "port=5001", "host=::"
-    ];
-    this.$tw.boot.boot(() => {
-
-
-      this.$tw.preloadTiddler({
-        text: "$protocol$//$host$" + "/mws/",
-        title: "$:/config/tiddlyweb/host",
-      });
-      // Boot the TW5 app
-      this.$tw.boot.boot(() => {
-        const Server = this.$tw.modules.execute("$:/core/modules/server/server.js", "tw-test.ts").Server;
-        this.twserver = new Server({
-          wiki: this.$tw.wiki,
-          variables: {
-            // do not use a trailing slash
-            "path-prefix": "/mws",
-            "root-tiddler": "$:/core/save/all",
-          },
-          routes: [],
-        });
-        this.$tw.hooks.invokeHook("th-server-command-post-start", this.twserver, null, "mws");
-      });
-
-    });
-  }
-}
-

package/src/server.ts

@@ -1,151 +0,0 @@
-import "./routes/router";
-import "./StateObject";
-import "./streamer";
-import "./global";
-import * as opaque from "@serenity-kit/opaque";
-import { existsSync, mkdirSync, PathOrFileDescriptor, readFileSync, writeFileSync } from 'node:fs';
-import * as sessions from "./routes/services/sessions";
-import * as attacher from "./routes/services/attachments";
-import { bootTiddlyWiki } from "./tiddlywiki";
-import { Commander } from "./commander";
-import { ListenerBase } from "./commands/mws-listen";
-import { createPasswordService } from "./routes/services/PasswordService";
-import { resolve } from "node:path";
-
-export * from "./routes/services/sessions";
-
-export interface MWSConfig {
-  /** 
-   * Listener config for the mws-listen command. 
-   * This is extracted from the config early and passed directly to mws-listen. 
-   */
-  listeners: {
-    /** The key file for the HTTPS server. If either key or cert are set, both are required, and HTTPS is enforced. */
-    key?: string
-    /** The cert file for the HTTPS server. If either key or cert are set, both are required, and HTTPS is enforced. */
-    cert?: string
-    /** The port to listen on. If not specified, listen will not be called. */
-    port?: number
-    /** The hostname to listen on. If this is specified, then port is required. */
-    host?: string
-  }[]
-  /**
-   * Called and awaited by the mws-listen command. 
-   * This is extracted from the config early and passed directly to mws-listen. 
-   */
-  onListenersCreated?: (listeners: ListenerBase[]) => Promise<void>;
-
-  /** 
-   * Enables the dev configuration of the server.
-   * The string is the absolute path to the dev repo. 
-   */
-  enableDevServer?: string
-  /** 
-   * Path or file descriptor to the password master key.
-   * If this key changes, all passwords will be invalid and need to be changed.
-   */
-  passwordMasterKeyFile: string | number
-  onPasswordKeyFileRead?: () => void;
-  /** Path to the datafolder. */
-  wikiPath: string
-  /** MWS site configuration */
-  config?: MWSConfigConfig
-  /** 
-   * The session manager class registers the login handler routes and sets the auth user 
-   */
-  SessionManager?: typeof sessions.SessionManager;
-  /** 
-   * The attachment service class is used by routes to handle attachments
-   */
-  AttachmentService?: typeof attacher.AttachmentService;
-
-}
-
-
-export interface MWSConfigConfig {
-  /** If true, allow users that aren't logged in to read. */
-  readonly allowAnonReads?: boolean
-  /** If true, allow users that aren't logged in to write. */
-  readonly allowAnonWrites?: boolean
-  /** If true, recipes will allow access to a user who does not have read access to all its bags. */
-  readonly allowUnreadableBags?: boolean
-  /** If true, files larger than `this * 1024` will be saved to disk alongside the database instead of in the database. */
-  readonly saveLargeTextToFileSystem?: number;
-  readonly enableGzip?: boolean
-  readonly enableBrowserCache?: boolean
-}
-
-
-
-/**
- * 
- * The promise returned will resolve once the commander has completed.
- * It will reject if the commander or anything else throws an error.
- * 
-```
-import startServer from "./src/server.ts";
-
-startServer({
-  // enableDevServer: true,
-  passwordMasterKey: "./localpass.key",
-  listeners: [{
-    key: "./localhost.key",
-    cert: "./localhost.crt",
-    port: 5000,
-  }],
-  config: {
-    wikiPath: "./editions/mws",
-    allowAnonReads: false,
-    allowAnonWrites: false,
-    allowUnreadableBags: false,
-  },
-});
-
-```
- */
-export default async function startServer(config: MWSConfig) {
-
-  await opaque.ready;
-
-  if (!["string", "number"].includes(typeof config.passwordMasterKeyFile)) {
-    throw new Error("passwordMasterKeyFile must be a string or number");
-  }
-
-  if (typeof config.passwordMasterKeyFile === "string"
-    && config.passwordMasterKeyFile
-    && !existsSync(config.passwordMasterKeyFile)
-  ) {
-    writeFileSync(config.passwordMasterKeyFile, opaque.server.createSetup());
-    console.log("Password master key created at", config.passwordMasterKeyFile);
-  }
-
-  const passwordMasterKey = readFileSync(config.passwordMasterKeyFile, "utf8").trim();
-
-  config.onPasswordKeyFileRead?.();
-
-  config.wikiPath = resolve(config.wikiPath);
-
-  const commander = new Commander(
-    config,
-    await bootTiddlyWiki(config.wikiPath),
-    await createPasswordService(passwordMasterKey)
-  );
-
-  await commander.init();
-  const cli = process.argv.slice(2);
-  // mws-command-separator prevents params after it from being applied to the command before it.
-  // it throws an error if it ends up with any params.
-  await commander.execute([
-    ...cli.length ? cli : [
-      "--mws-init-store",
-      "--mws-listen"
-    ],
-  ]);
-
-  if (commander.setupRequired) {
-    console.log("MWS setup required. Please run either --mws-init-store or --mws-load-archive");
-    process.exit(1);
-  }
-
-}
-