@lat-murmeldjur/weeb_3 0.0.295001

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 lat-murmeldjur
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,205 @@
+# Weeb-3 - A Swarm client for browsers
+
+This project is a work in progress swarm client implementation relying solely on browser side technologies.
+It uses [wasm-pack](https://rustwasm.github.io/docs/wasm-pack/) to build the project for use in the browser.
+
+## Building the code
+
+Ensure you have [`wasm-pack`](https://rustwasm.github.io/wasm-pack/), [`protoc`](https://grpc.io/docs/protoc-installation/), and [`clang`](https://clang.llvm.org/) installed.
+
+1. Build the client library:
+```shell
+
+RUSTFLAGS='--cfg getrandom_backend="wasm_js"' wasm-pack build --target web --out-dir static --out-name weeb_3
+```
+
+2. Start the local server to serve html, js and wasm files:
+```shell
+cargo run
+```
+Note this server uses an unsecure self-signed certificate to provide https, which is not sufficient to enable Service Workers in chrome etc. This enables displaying single files from swarm, however to display websites a service worker is necessary, which requires a certificate deemed safe by the browser. You can however get your own safe certificate from - for example - github pages by forking the repository and setting the github pages to 'docs', and copying your latest version of the files from the static folder to the docs folder. 
+
+3. Open the URL (https://localhost:8080/weeb-3 or for the github pages hosted version https://lat-murmeldjur.github.io/weeb-3)
+
+## Using the npm package
+
+The wasm-pack build now prepares the generated `static/package.json` for publishing to npm together with the assets required by the wrapper:
+
+- `static/snippets/web3-0742d85b024bb6f5/inline0.js`
+- `static/weeb_3.js`
+- `static/weeb_3_bg.wasm`
+
+After publishing, the package can be used with the same API shape as `static/example.html`:
+
+```js
+import init, { SekireiNo103, BootstrapNode } from "@lat-murmeldjur/weeb_3";
+
+await init();
+
+const weeb3node = new SekireiNo103();
+```
+
+The workflow defaults to publishing under the GitHub repository owner scope. If you need a different npm scope, set the `NPM_SCOPE` repository variable in GitHub Actions before pushing to `main`.
+
+## Automated publishing
+
+The GitHub Actions workflow rebuilds the wasm package from `static`, rewrites the generated `static/package.json` into the publishable npm shape, checks the tarball with `npm pack --dry-run`, and publishes it on every push to `main`.
+
+Published versions are rewritten to unique stable semver numbers derived from the Cargo version plus the GitHub Actions run number and retry attempt, so each `main` publish gets a new release version without hand-editing `package.json`.
+
+To make the publish step fully automatic, configure npm trusted publishing for this repository and the `plain.yml` workflow.
+
+If you prefer a token fallback instead, add a repository secret named `NPM_TOKEN`.
+
+## [Notes]
+
+### Compatibility - Supported Browsers
+
+- Chrome (on Windows 11)
+- Chrome (Android)
+- Brave (on Windows 11)
+- Edge 
+- Firefox (on Windows 11)
+- Firefox (on Android)
+
+Testing and modifying for other browsers is planned.
+
+### How it works (architectural overview)
+
+The weeb-3 client consists of several logical components:
+- The web interface (implemented by src/interface.rs & of course static/index.html) that creates with the main weeb process and loads the service worker
+- The libp2p/swarm client (with main entry point in src/lib.rs)
+- A service worker, that enables hot-loading assets on relative paths for websites loaded from swarm (found in static/service.js)
+
+Below is a piece by piece overview of the current logic of components
+
+#### The interface
+
+As of "commit number 189" the instantiator of all components is the index.html that starts the interface (by calling the function "interweeb" from src/interface.rs).
+
+The interweeb function has the following roles (in order of appearing in the code):
+- Starting the libp2p/swarm client in an async block
+- Setting a listener on the settings text input fields, triggering changing bootnode and connection settings to the weeb process on a button based event
+- Setting a listener on the navigation text input field, triggering requests to the shared worker on content change of the navigation text input field 
+- Setting a listener on the create storage input fields, triggering connecting metamask and buying a batch with provided parameters
+- Setting a listener on the reuse space button, triggering resetting an already existing batch to clean slate
+- Setting a listener on the upload input fields, triggering uploading with the existing batch if there is one
+- Listening to logs sent back by the weeb process, as well as resources and responses of triggered requests and displaying them 
+- Starting connection to bootnode after 600 milliseconds
+
+#### The weeb process
+
+Originally implemented in a shared web worker, the main weeb process was intended to function as a common resource for multiple tabs of the same origin.
+This design would have enabled having one swarm client serving multiple open tabs, however, at the cost of giving up on the possibility of being able to open webrtc connections. However, chrome on android does not support shared workers, so for the time being, the main weeb process was transfered back into the tab.
+
+The high level architecture of the client resides in src/lib.rs, which (in order of appearing in the code) implements the following functions:
+
+- Defining and importing the swarm protocols generated by the protoc compiler
+- Defining the client (as the class Sekirei), it's in-memory registry of peers and their accounting (as the struct Wings)
+- Defining the 6 main functions of the client, namely 
+	1) Changing the bootnode address and network id
+	2) Uploading a file or a tar based collection, optionally to a feed as well
+	3) A function that enables using the running client to retreive resources (the function "acquire")
+	4) A function to reset a postage stamp to original state
+	5) Instantiation (the "new" function), that starts the libp2p client 
+	6) A function that continues running the client, asynchronously maintains/establishes connections, serves requests from the interface, and engages in protocols with the swarm (the "run" function)
+	7) Helper functions for the interface to get new log lines, connection numbers, and to submit new logs meant to be shown on the interface
+
+In slightly more detail, the new function does the following (in order of appearing in the code):
+- Randomises a new secret keypair
+- Starts a libp2p client with the stream libp2p behaviour enabled using webrtc transport
+- Creates a registry of peers (connected_peers, overlay_peers) and peer accounting (accounting_peers, ongoing_refreshments)
+- Creates a message port (to be listened to by the client and to be used by the acquire function)
+This message port can receive the writing end of a channel of bytes along with an address, so that it can write back the results of looking up the address to the channel received.
+
+The run function of the client implements an asynchronous architecture that does the following functions (in order of appearance in the code):
+- Creating channels for swarm specific functions 
+	1) Receiving new peers to connect from the gossip protocol (peers_instructions_chan, connections_instructions_chan)
+	2) Accounting related functions (accounting_peer_chan, pricing_chan, refreshment_instructions_chan, refreshment_chan)
+	3) Receiving new bootnode address to connect to
+- Setting up listening to gossip protocol messages (information about existing peers) and pricing protocol messages (for receiving connected peers payment threshold updates)
+- An async routine to continously establish new libp2p-connections (dial) and consume libp2p-swarm events (swarm_event_handle) as well as dialing to bootnode
+- An async routine that wraps a number of further async routines for the following functions (event_handle):
+	1) Accounting connecting newly established peer connections (k1)
+	2) Setting payment thresholds for peers after successfully receiving payment threshold updates in the pricing protocol (k2)
+	3) Initiating refreshments/pseudosettle protocol for peers when triggered by accounting actions (k3)
+	4) Registering the results of successful refreshments towards peers (k4)
+- Two async routines that listens to high level download / upload requests 
+- Two async routines that listens to data object level download / upload requests enabling joining and splitting of chunks
+- Two async routines that enable concurrent chunk level pushsync and retrieval requests
+- An async routine that attempts to conduct handshakes with dialed connections
+
+Currently - due to the blocking - non-blocking nature of the async framework, and to avoid a waiting thread hogging the single execution thread, the aforementioned routines intermittently try progressing every 600ms with non cpu intensive async sleeps happening in-between.
+
+#### The Swarm Client Subcomponents
+
+The aforementioned architecture further depends on the following code modules:
+- The protocol handlers for handshake, hive, pricing, pseudosettle and retrieval (src/handlers.rs)
+- The accounting functions, such as calculating chunk prices, reserving, crediting, refreshing (src/accounting.rs)
+- The retrieval logic such as selecting peers to retrieve chunks from, decrypting chunks, joining files and triggering manifest interpretations (src/retrieval.rs)
+- The pushsync logic such as selecting peers to push chunks to, encrypting chunks, splitting files, triggering manifest and soc creation (src/upload.rs)
+- The manifest creation logic (src/manifest_upload.rs)
+- The manifest interpretation logic (src/manifest.rs)
+- The ENS contenthash resolution logic (src/ens.rs)
+- The indexeddb in-browser storage solution (src/persistence.rs)
+- Common methods and struct declarations including DOM manipulation, calculating proximity orders, validating content addressed and single owner chunks, calculating feed addresses, and encoding/decoding resource groups to communicate through byte channels e.g. towards the interface (src/conventions.rs)
+
+#### Persistence and identity
+
+The weeb process persists 3 types of data:
+- Caching chunks retrieved previously
+- Identity related keys and identifiers used for uploads and creating feeds
+	1) Batch ID
+	2) Private key of Batch Owner
+	3) Batch Bucket Limit
+	4) Private key of Feed Owner
+- Saturation of individual Batch Buckets
+
+The indexeddb access is denied to loaded websites by opening websites from swarm in iframes marked with the sandbox attribute.
+The private keys are not used for blockchain purposes, the wallet responsible for buying a batch can only be connected through metamask currently.
+The libp2p node keys are chosen randomly each time the tab is reloaded, resulting in a unique overlay every time
+
+### The Service Worker
+
+Quoting from the [MDN documentation](https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API), "Service workers essentially act as proxy servers that sit between web applications, the browser, and the network (when available). They are intended, among other things, to enable the creation of effective offline experiences, intercept network requests, and take appropriate action based on whether the network is available, and update assets residing on the server. They will also allow access to push notifications and background sync APIs.".
+
+The weeb-3 interface functions can display single files, for example pictures, documents, and other single files without relying on the service worker through dynamically creating blobs with associated mime types from the data retrieved from swarm, and making them available on [virtual urls](https://developer.mozilla.org/en-US/docs/Web/API/URL/createObjectURL_static). These resources are displayed in embed tags prepended to the content of the resultField html tag.
+
+However, this createObjectUrl method inserts random strings into the virtual urls assigned to individual resources, which makes it unfeasible to be used to render complete websites, as relative paths of assets embedded in the site would be broken by such random strings. This necessitates the use of a service worker, which can intercept the http requests aimed towards the de facto server (for example github pages) and is able to create objects with deterministic url paths to be served in response to these requests, making serving relative assets possible.
+
+To enable this, upon detecting a website manifest, the interface sends each retrieved resource complete with relative path and mime type to the service worker (the message event listener in static/service.js), which injects it into a named cache ('default0'), before prepending the website index document as an iframe to the resultField html tag of the weeb-3 browser tab. 
+
+This service worker is only enabled by the browser if the browser detects that the site is served through a secure https connection based on a trusted certificate, as this functionality is clearly a security sensitive asset that can be used to intercept http requests and inject arbitrary resources. The use of this functionality also creates new surfaces of attack such as creating malicious websites that could load a malevolent service worker at runtime, to enable further malicious injections. Disableing such attacks is part of the security development topic of the planned developments section.
+
+### Main dependencies
+
+The weeb-3 project uses the following main rust crates:
+- libp2p
+- alloy
+- ethers
+- web3-rs
+- async-std
+- wasm-bindgen
+- js-sys
+- web-sys
+- indexed_db_futures
+
+### Concurrency and memory limitations
+
+The architecture of the weeb process enables a high level of concurrency between different tasks, sending a high number of different types of protocol messages parallelly. The webassembly architecture currently does not support threads or utilizing multiple CPUs, and the 32bit memory addressing scheme limits the usable memory to 4 GBs. Offloading the work to multiple non-specialized web-workers would increase this limit in both dimensions as each web worker has a separate memory address space and a separate physical thread.
+
+## [Planned development]
+
+- Adding functionality to the service worker to enable triggering requests towards the shared worker, to retrieve resources when swarm references are present in a website, alternatively, achieving the same by overwriting navigation bar contents when an onclick event is detected to be a swarm reference
+- Simultaneous manifest fork lookups  
+- Multi-threading through web-workers
+- Adding the swarm ACT feature
+- Wallet related functionality such as using cheques 
+- Penetration testing against service worker replacement and other injection types of attacks
+- Penetration testing loaded websites access to keys in indexeddb / loading single executable files
+- Refinements in error propagation, reliability, robustness, status updates in ongoing processes
+
+
+
+
+

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@lat-murmeldjur/weeb_3",
+  "type": "module",
+  "description": "A Swarm client for browsers",
+  "version": "0.0.295001",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/lat-murmeldjur/weeb-3.git"
+  },
+  "main": "./weeb_3.js",
+  "homepage": "https://github.com/lat-murmeldjur/weeb-3#readme",
+  "types": "./weeb_3.d.ts",
+  "module": "./weeb_3.js",
+  "bugs": {
+    "url": "https://github.com/lat-murmeldjur/weeb-3/issues"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "files": [
+    "snippets/web3-0742d85b024bb6f5/inline0.js",
+    "weeb_3.d.ts",
+    "weeb_3.js",
+    "weeb_3_bg.wasm",
+    "weeb_3_bg.wasm.d.ts"
+  ],
+  "sideEffects": [
+    "./snippets/*"
+  ],
+  "keywords": [
+    "bee",
+    "browser",
+    "swarm",
+    "wasm",
+    "webassembly"
+  ],
+  "exports": {
+    ".": {
+      "types": "./weeb_3.d.ts",
+      "import": "./weeb_3.js",
+      "default": "./weeb_3.js"
+    }
+  }
+}

package/snippets/web3-0742d85b024bb6f5/inline0.js

@@ -0,0 +1 @@
+export function get_provider_js() {return window.ethereum}

package/weeb_3.d.ts

@@ -0,0 +1,131 @@
+/* tslint:disable */
+/* eslint-disable */
+
+export class BootstrapNode {
+    free(): void;
+    [Symbol.dispose](): void;
+    constructor(multiaddr: string, usable: boolean);
+    readonly multiaddr: string;
+    readonly usable: boolean;
+}
+
+export class RequestArguments {
+    private constructor();
+    free(): void;
+    [Symbol.dispose](): void;
+    readonly method: string;
+    readonly params: Array<any>;
+}
+
+export class Sekirei {
+    private constructor();
+    free(): void;
+    [Symbol.dispose](): void;
+    acquire(address: string): Promise<Uint8Array>;
+    change_bootnode_address(address: string, _id: string, usable_in_protocols: boolean): Promise<Uint8Array>;
+    get_connections(): Promise<bigint>;
+    get_current_logs(): Promise<string[]>;
+    get_ongoing_connections(): Promise<bigint>;
+    interface_log(log0: string): void;
+    static new(_st: string): Sekirei;
+    post_push_chunk(d: Uint8Array, soc: boolean, chunk_address: Uint8Array, stamp: Uint8Array): Promise<Uint8Array>;
+    post_upload(file: File, encryption: boolean, index_string: string, add_to_feed: boolean, feed_topic: string): Promise<Uint8Array>;
+    reset_stamp(): Promise<Uint8Array>;
+    run(_st: string): Promise<void>;
+}
+
+export class SekireiNo103 {
+    free(): void;
+    [Symbol.dispose](): void;
+    constructor();
+    postPushChunk(data: Uint8Array, soc: boolean, chunk_address: Uint8Array, stamp: Uint8Array): Promise<string>;
+    resetStamp(): Promise<object>;
+    retrieve(address: string): Promise<Array<any>>;
+    start(bootstrap_nodes: BootstrapNode[], network_id: string): void;
+    upload(file: File, encryption: boolean, index_string: string, add_to_feed: boolean, feed_topic: string): Promise<object>;
+}
+
+export class Wings {
+    private constructor();
+    free(): void;
+    [Symbol.dispose](): void;
+}
+
+export function interweeb(_st: string): Promise<void>;
+
+export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
+
+export interface InitOutput {
+    readonly memory: WebAssembly.Memory;
+    readonly __wbg_bootstrapnode_free: (a: number, b: number) => void;
+    readonly __wbg_sekireino103_free: (a: number, b: number) => void;
+    readonly bootstrapnode_multiaddr: (a: number) => [number, number];
+    readonly bootstrapnode_new: (a: number, b: number, c: number) => number;
+    readonly bootstrapnode_usable: (a: number) => number;
+    readonly sekireino103_new: () => number;
+    readonly sekireino103_postPushChunk: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => any;
+    readonly sekireino103_resetStamp: (a: number) => any;
+    readonly sekireino103_retrieve: (a: number, b: number, c: number) => any;
+    readonly sekireino103_start: (a: number, b: number, c: number, d: number, e: number) => void;
+    readonly sekireino103_upload: (a: number, b: any, c: number, d: number, e: number, f: number, g: number, h: number) => any;
+    readonly interweeb: (a: number, b: number) => any;
+    readonly __wbg_sekirei_free: (a: number, b: number) => void;
+    readonly __wbg_wings_free: (a: number, b: number) => void;
+    readonly sekirei_acquire: (a: number, b: number, c: number) => any;
+    readonly sekirei_change_bootnode_address: (a: number, b: number, c: number, d: number, e: number, f: number) => any;
+    readonly sekirei_get_connections: (a: number) => any;
+    readonly sekirei_get_current_logs: (a: number) => any;
+    readonly sekirei_get_ongoing_connections: (a: number) => any;
+    readonly sekirei_interface_log: (a: number, b: number, c: number) => void;
+    readonly sekirei_new: (a: number, b: number) => number;
+    readonly sekirei_post_push_chunk: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => any;
+    readonly sekirei_post_upload: (a: number, b: any, c: number, d: number, e: number, f: number, g: number, h: number) => any;
+    readonly sekirei_reset_stamp: (a: number) => any;
+    readonly sekirei_run: (a: number, b: number, c: number) => any;
+    readonly __wbg_requestarguments_free: (a: number, b: number) => void;
+    readonly requestarguments_method: (a: number) => [number, number];
+    readonly requestarguments_params: (a: number) => any;
+    readonly wasm_bindgen__convert__closures_____invoke__h06a7e7b2a563bd9a: (a: number, b: number, c: any) => [number, number];
+    readonly wasm_bindgen__convert__closures_____invoke__h2b3586c1e5138808: (a: number, b: number, c: any) => [number, number];
+    readonly wasm_bindgen__convert__closures_____invoke__h88996e8058e7b4cf: (a: number, b: number, c: any, d: any) => void;
+    readonly wasm_bindgen__convert__closures_____invoke__h2c2b25660306fe52: (a: number, b: number, c: any) => void;
+    readonly wasm_bindgen__convert__closures_____invoke__h592ace81aa415f10: (a: number, b: number, c: any) => void;
+    readonly wasm_bindgen__convert__closures_____invoke__h592ace81aa415f10_3: (a: number, b: number, c: any) => void;
+    readonly wasm_bindgen__convert__closures_____invoke__hb7695be2fdd72d20: (a: number, b: number, c: any) => void;
+    readonly wasm_bindgen__convert__closures_____invoke__h592ace81aa415f10_6: (a: number, b: number, c: any) => void;
+    readonly wasm_bindgen__convert__closures_____invoke__hb28937c0609bdd79: (a: number, b: number) => void;
+    readonly wasm_bindgen__convert__closures_____invoke__h294687bdb60910fc: (a: number, b: number) => void;
+    readonly wasm_bindgen__convert__closures_____invoke__h41947166d8d508a5: (a: number, b: number) => void;
+    readonly __wbindgen_malloc: (a: number, b: number) => number;
+    readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
+    readonly __wbindgen_exn_store: (a: number) => void;
+    readonly __externref_table_alloc: () => number;
+    readonly __wbindgen_externrefs: WebAssembly.Table;
+    readonly __wbindgen_destroy_closure: (a: number, b: number) => void;
+    readonly __externref_drop_slice: (a: number, b: number) => void;
+    readonly __wbindgen_free: (a: number, b: number, c: number) => void;
+    readonly __externref_table_dealloc: (a: number) => void;
+    readonly __wbindgen_start: () => void;
+}
+
+export type SyncInitInput = BufferSource | WebAssembly.Module;
+
+/**
+ * Instantiates the given `module`, which can either be bytes or
+ * a precompiled `WebAssembly.Module`.
+ *
+ * @param {{ module: SyncInitInput }} module - Passing `SyncInitInput` directly is deprecated.
+ *
+ * @returns {InitOutput}
+ */
+export function initSync(module: { module: SyncInitInput } | SyncInitInput): InitOutput;
+
+/**
+ * If `module_or_path` is {RequestInfo} or {URL}, makes a request and
+ * for everything else, calls `WebAssembly.instantiate` directly.
+ *
+ * @param {{ module_or_path: InitInput | Promise<InitInput> }} module_or_path - Passing `InitInput` directly is deprecated.
+ *
+ * @returns {Promise<InitOutput>}
+ */
+export default function __wbg_init (module_or_path?: { module_or_path: InitInput | Promise<InitInput> } | InitInput | Promise<InitInput>): Promise<InitOutput>;