ai-nevermore 0.1.0 → 0.1.2

package/README.md

@@ -3,6 +3,22 @@ Nevermore
 
 Nevermore is a library to obfuscate media on the web to prevent AI scraping.
 
+The following table is a matrix of potential techniques and media:
+
+|              | Random Poison  | Targeted Poison | Obfuscation   | Encryption |
+|--------------|----------------|-----------------|---------------|------------|
+| Text         | Nevermore      | Nevermore[TBD]  | Nevermore     |            |
+| Image        |                | Nightshade      | Nevermore     |            |
+| Video        | Nevermore[TBD] |                 | Nevermore[TBD]|            |
+| Audio        |                | HarmonyCloak    |               |            |
+| 3D Models    | Nevermore[TBD] |                 | Nevermore[TBD]|            |
+| Code         | Nevermore[TBD] |                 |               |            |
+
+There are conceptually 2 other techniques to fight AI.
+
+1) Scrape traversal traps: dynamically generated nonsense hierarchy of pages to indefinitely trap the scraper on your site. It incurs additional hosting costs, but gives the possibility of being blacklisted by the scraper. Examples of this include [Nepenthes](https://zadzmo.org/code/nepenthes/) and [Iocaine](https://iocaine.madhouse-project.org)
+2) Scrape Poison: This technique exploits weaknesses in the scrape engines or or selector format to prevent scraping of the correct data. This is currently only theoretical, but is something we are looking at.
+
 Text
 ---
 while a user will see
@@ -130,8 +146,25 @@ Roadmap
 - [x] image encoding
 - [x] web component decoder
 - [x] markup obfuscation
+- [ ] windows compatibility
 - [ ] self randomizing dictionary
 - [ ] add a replacement mode (opposed to a tokenizer based solution)
+- [ ] bindings
+    - [ ] PHP CL binding
+    - [ ] Python CL binding
+    - [ ] Java CL binding
+    - [ ] Ruby CL binding
+- [ ] native CMS Support
+    - [ ] KeystoneJS
+    - [ ] Ghost
+    - [ ] Strapi
+- [ ] other CMS Support
+    - [ ] wordpress
+- [ ] Framework Support
+    - [ ] express
+    - [ ] next.js
+    - [ ] laravel
+    - [ ] rails
 
 Development
 -----------

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "ai-nevermore",
-    "version": "0.1.0",
+    "version": "0.1.2",
     "keywords": [
         "adversarial",
         "content",
@@ -22,6 +22,9 @@
         "./image": {
             "import": "./src/image.mjs"
         },
+        "./html": {
+            "import": "./src/html.mjs"
+        },
         "./encoded-image": {
             "import": "./src/encoded-image-component.mjs"
         },

package/src/html.mjs

@@ -1,10 +1,10 @@
 import { computeIndexKeys, generateHTMLAndCSS } from '../src/index.mjs';
 import { NevermoreImage } from '../src/image.mjs';
 
-export const transformHTML = async (node, idx)=>{
+export const transformHTML = async (node, idx, textureDir)=>{
     let content = '';
     if(node.childNodes) for(let lcv=0; lcv < node.childNodes.length; lcv++){
-        content += await transformHTML(node.childNodes[lcv]);
+        content += await transformHTML(node.childNodes[lcv], idx, textureDir);
     }
     
     try{
@@ -23,12 +23,16 @@ export const transformHTML = async (node, idx)=>{
                 break;
             case 'img':
                 const srcAttr = node.attrs.find((attr)=> attr.name.toLowerCase() === 'src');
-                const image = new NevermoreImage({ url: srcAttr.value, maskDir:'./textures'});
+                const image = new NevermoreImage({ 
+                    url: srcAttr.value, 
+                    maskDir: textureDir ||'./textures'
+                });
                 await image.ready;
                 const canvas = image.encode();
                 const url = canvas.toDataURL('jpg');
                 //return `<img src="${url}" ></img>`;
                 return `<encoded-image src="${url}" key="${image.key}"></encoded-image>`;
+                break;
             case 'head':  
                 const att = node.attrs?' '+node.attrs.map((attr)=>`${attr.name}="${attr.value}"`).join(' '):'';
                 return `<${node.tagName}${att}>${content}<script type="importmap">
@@ -47,7 +51,8 @@ export const transformHTML = async (node, idx)=>{
                 </script>
                 <script type="module">
                     import 'nevermore/encoded-image-component';
-                </script></${node.tagName}>`
+                </script></${node.tagName}>`;
+                break;
             default: 
                 const attrs = node.attrs?' '+node.attrs.map((attr)=>`${attr.name}="${attr.value}"`).join(' '):'';
                 if(node.tagName) return `<${node.tagName}${attrs}>${content}</${node.tagName}>`

package/src/image.mjs

@@ -79,60 +79,63 @@ export class NevermoreImage{
         )) throw new Error('a set of masks are required');
         this.textureDictionary = options.dictionary;
         this.ready = new Promise(async (resolve, reject)=>{
-            // load all masks
-            // TODO: load from cache
-            let masks = options.masks;
-            const textureWork = [];
-            const textureDictionary = {};
-            if(options.maskDir){
-                const types = ['jpg', 'jpeg', 'gif', 'png'];
-                const files = (await File.list(options.maskDir)).filter((name)=>{
-                    const parts = name.split('.');
-                    const ext = parts.pop().toLowerCase();
-                    return types.indexOf(ext) !== -1;
-                    // TODO: support mime through magic numbers
-                });
-                const fileLoads = [];
-                for(let lcv=0; lcv<files.length; lcv++){
-                    const pth = new Path(files[lcv]);
-                    if(!pth.parsed) throw new Error('path not parsed');
-                    const parsed = pth.parsed.posix || pth.parsed.win32;
-                    const canvasLoad = Canvas.load(
-                        Path.join(options.maskDir, files[lcv])
-                    );
-                    textureWork.push(new Promise(async (resolve)=>{
-                        resolve({
-                            name: parsed.name,
-                            canvas: await canvasLoad
-                        });
-                    }))
-                    fileLoads.push(Canvas.load(Path.join(options.maskDir, files[lcv])));
+            try{
+                // load all masks
+                // TODO: load from cache
+                let masks = options.masks;
+                const textureWork = [];
+                const textureDictionary = {};
+                if(options.maskDir){
+                    const types = ['jpg', 'jpeg', 'gif', 'png'];
+                    const files = (await File.list(options.maskDir)).filter((name)=>{
+                        const parts = name.split('.');
+                        const ext = parts.pop().toLowerCase();
+                        return types.indexOf(ext) !== -1;
+                        // TODO: support mime through magic numbers
+                    });
+                    const fileLoads = [];
+                    for(let lcv=0; lcv<files.length; lcv++){
+                        const pth = new Path(files[lcv]);
+                        if(!pth.parsed) throw new Error('path not parsed');
+                        const parsed = pth.parsed.posix || pth.parsed.win32;
+                        const canvasLoad = Canvas.load(
+                            Path.join(options.maskDir, files[lcv])
+                        );
+                        textureWork.push(new Promise(async (resolve)=>{
+                            resolve({
+                                name: parsed.name,
+                                canvas: await canvasLoad
+                            });
+                        }))
+                        fileLoads.push(Canvas.load(Path.join(options.maskDir, files[lcv])));
+                    }
+                    masks = await Promise.all(fileLoads);
+                    const texturesLoaded = await Promise.all(textureWork);
+                    for(let lcv=0; lcv<texturesLoaded.length; lcv++){
+                        textureDictionary[
+                            texturesLoaded[lcv].name
+                        ] = texturesLoaded[lcv].canvas;
+                    };
+                    this.textureDictionary = textureDictionary;
                 }
-                masks = await Promise.all(fileLoads);
-                const texturesLoaded = await Promise.all(textureWork);
-                for(let lcv=0; lcv<texturesLoaded.length; lcv++){
-                    textureDictionary[
-                        texturesLoaded[lcv].name
-                    ] = texturesLoaded[lcv].canvas;
-                };
-                this.textureDictionary = textureDictionary;
-            }
-            this.masks = masks;
-            this.canvas = options.canvas;
-            // masks loaded, now load the base image
-            if(options.url){
-                this.canvas = await Canvas.load(options.url);
-            }
-            if(!this.canvas){
-                return reject(new Error(
-                    'no base image provided to Image File'
-                ));
+                this.masks = masks;
+                this.canvas = options.canvas;
+                // masks loaded, now load the base image
+                if(options.url){
+                    this.canvas = await Canvas.load(options.url);
+                }
+                if(!this.canvas){
+                    return reject(new Error(
+                        'no base image provided to Image File'
+                    ));
+                }
+                //make an id
+                this.key = options.key || `${makeKey(5)}-${makeKey(5)}-${makeKey(5)}`;
+                //we're ready to do work
+                resolve();
+            }catch(ex){
+                reject(ex);
             }
-            //make an id
-            this.key = options.key || `${makeKey(5)}-${makeKey(5)}-${makeKey(5)}`;
-            //console.log('KEY:', this.key);
-            //we're ready to do work
-            resolve();
         });
     }
     

package/src/index.mjs

@@ -71,26 +71,26 @@ export const computeIndexKeys = async (textBody)=>{
     const parser = new ParseEnglish();
     const node = parser.parse(textBody);
     const index = {};
+    const skip = ['.']
     await traverse(node, async (thisNode)=>{
         let res = null;
-        if(thisNode.value){
+        if(thisNode.value && thisNode.type === 'TextNode'){
+            //if(skip.indexOf() !== -1) return;
             try{
                 res = await define(thisNode.value);
-            }catch(ex){}
+            }catch(ex){ }
         }
         if(thisNode.type === 'TextNode' && res){
             thisNode.word = res.word;
             thisNode.types = res.types;
             thisNode.definitions = res.definitions;
             thisNode.synonyms = res.synonyms;
-            //const thesres = await thesaurus.lookup(thisNode.word);
             if(
                 thisNode.types.indexOf('verb') !== -1 || 
                 thisNode.types.indexOf('noun') !== -1 
             ){
                 thisNode.replacement = await randomWord();
                 index[thisNode.word] = thisNode.replacement;
-                //index[thisNode.word] = thisNode.synonyms[0];
             }
         }
     });