@xriss/weerss 0.0.2

package/hoard.js

@@ -0,0 +1,85 @@
+
+
+let hoard={}
+export default hoard
+
+
+
+import      db        from "./db.js"
+
+
+
+// make sure we have a cache of the page available, no matter how old
+hoard.first_text=async function(url)
+{
+	if( ! await db.get("hoard",url) ) // if not exist
+	{
+		return await hoard.fetch_text(url) // make exist
+	}
+}
+
+hoard.fast_text=async function(url)
+{
+	let it
+	it=await db.get("hoard",url)
+	if(it)
+	{
+		let randage=it.randage||0	// make updates have a random interval
+		if( ( Date.now() - (new Date(it.date)).getTime() ) < ( hoard.maxage + randage ) ) // use cache ?
+		{
+			return it.text // no refresh
+		}
+		hoard.fetch_text(url,true) // refresh
+		return it.text // but return fast
+	}
+	else
+	{
+		return await hoard.fetch_text(url)
+	}
+}
+
+// cache lasts from maxage to double maxage as a random thingy
+hoard.maxage=15*60*1000
+
+hoard.maxsize=2*1024
+
+hoard.fetch_text=async function(url,refresh)
+{
+//	let corsurl=(arss.cors||"")+url
+	let corsurl=url
+	let oldtext
+	let it
+	if(!refresh) { it=await db.get("hoard",url) } // try cache
+	if(it)
+	{
+		let randage=it.randage||0	// make updates have a random interval
+		if( ( Date.now() - (new Date(it.date)).getTime() ) < ( hoard.maxage + randage ) ) // use cache ?
+		{
+			return it.text
+		}
+		oldtext=it.text
+	}
+	let write=async function()
+	{
+		it={}
+		it.status=0
+		it.date=new Date()
+		try{
+
+			let res=await Promise.race([
+				fetch(corsurl),
+				new Promise((_, reject) => setTimeout(() => reject("timeout"), 2*1000)),
+			])
+			it.status=res.status
+			it.text=await res.text()
+			
+			if(it.text=="undefined") { it.text="STATUS : "+it.status }
+			it.randage=Math.floor(Math.random() * hoard.maxage);
+			await db.set("hoard",url,it) // always write if we get here, errors will not write
+
+		}catch(e){console.error("failed url",url);console.error(e)}
+	}
+	await write()
+	return it.text
+}
+

package/items.js

@@ -0,0 +1,188 @@
+
+
+let items={}
+export default items
+
+
+import      hoard     from "./hoard.js"
+import      db        from "./db.js"
+import      jxml      from "./jxml.js"
+
+import { configure } from 'safe-stable-stringify'
+const stringify = configure({})
+
+
+
+
+items.cached={}
+
+items.precache=async function() // faster then individual cache, db is fecking slow
+{
+	for(let item of await db.list("items"))
+	{
+		items.cache(item.uuid,item)
+	}
+}
+
+
+items.cache=async function(uuid,item) // probably fast
+{
+	if(item) { items.cached[uuid]=item ; return item }
+	if(items.cached[uuid]) { return items.cached[uuid] }
+	await items.get(uuid)
+	return items.cached[uuid]
+}
+items.get=async function(uuid) // always slow
+{
+	let item=await db.get("items",uuid)
+	if(item) { items.cache(item.uuid,item) }
+	return item
+}
+items.set=async function(item)
+{
+	await db.set("items",item.uuid,item)
+	if(item) { items.cache(item.uuid,item) }
+}
+
+
+
+items.prepare=function(item,feed)
+{
+	let atom=item.atom||{}
+	let rss=item.rss||{}
+	let any=item.rss||item.atom||{}
+
+	if(feed) // cache some basic feed values
+	{
+		item.feed=feed.url
+		item.feed_title=feed.title
+		item.feed_tags=feed.tags
+	}
+
+	if(!item.date) // do not change date, just set it once
+	{
+		if(rss["/pubdate"])
+		{
+			item.date=new Date(rss["/pubdate"])
+		}
+		else
+		if(atom["/published"]) // prefer published
+		{
+			item.date=new Date(atom["/published"])
+		}
+		else
+		if(atom["/updated"]) // but sometimes we do not have it
+		{
+			item.date=new Date(atom["/updated"])
+		}
+		else
+		if(atom["/created"]) // wtf?
+		{
+			item.date=new Date(atom["/created"])
+		}
+		else
+		if(atom["/issued"]) // wtf?
+		{
+			item.date=new Date(atom["/issued"])
+		}
+		else
+		if(atom["/modified"]) // wtf?
+		{
+			item.date=new Date(atom["/modified"])
+		}
+		else
+		if(any["/dc:date"]) // why you do dis?
+		{
+			item.date=new Date(any["/dc:date"])
+		}
+		else
+		if(feed && feed.date) // maybe the feed had a date
+		{
+			item.date=feed.date
+		}
+	}
+
+
+	if(rss["/link"])
+	{
+		item.link=rss["/link"]
+	}
+	else
+	if(rss["/id"]) // sometimes the real link is the RSS id
+	{
+		item.link=rss["/id"]
+	}
+	else
+	if(atom["/link"])
+	{
+		item.link=atom["/link"][0]["@href"]
+		for(let link of atom["/link"])
+		{
+			if( link["@rel"]=="alternate" )
+			{
+				item.link=link["@href"]
+			}
+		}
+	}
+	else
+	if(atom["/id"]) // sometimes the real link is the RSS id
+	{
+		item.link=atom["/id"]
+	}
+
+	item.title=item.title||""
+	if(rss["/title"])
+	{
+		item.title=rss["/title"]
+	}
+	else
+	if(atom["/title"])
+	{
+		item.title=atom["/title"]
+	}
+
+	item.html=item.html||""
+	if(rss["/description"])
+	{
+		item.html=rss["/description"]
+	}
+	else
+	if(atom["/content"])
+	{
+		item.html=atom["/content"]
+	}
+
+	// prefer link, but might be a feed url + ( id or date )
+	item.uuid=item.link || ( item.feed+"#"+( ( rss["/guid"] || atom["/id"] ) || (""+item.date) ) )
+
+	return item
+}
+
+items.add_count=0
+items.add=async function(it)
+{
+	let old=await items.cache(it.uuid) // db.get("items",it.uuid)
+	let item={}
+	if(old)
+	{
+		for(let n in old ){ item[n]=old[n] }
+	}
+	else
+	{
+		items.add_count++
+	}
+	for(let n in it )
+	{
+		item[n]=item[n]||it[n]
+	}
+	if(old) // old date has precedence
+	{
+		item.date=old.date||item.date
+	}
+	item.date=item.date||new Date() // make sure we have a date
+
+	if( stringify(old) != stringify(item) ) // something changed
+	{
+		await items.set(item)
+	}
+}

package/jxml.js

@@ -0,0 +1,262 @@
+
+let jxml={}
+export default jxml
+
+
+import sax from "sax"
+
+
+jxml.xmap={}
+
+// technically we should keep nesting outlines
+jxml.xmap.opml={
+	"/opml/body/outline":true,
+	"/opml/body/outline/outline":true,
+	"/opml/body/outline/outline/outline":true,
+	"/opml/body/outline/outline/outline/outline":true,
+}
+
+jxml.xmap.rss={
+	"/rss/channel/category":true,
+	"/rss/channel/item":true,
+	"/rss/channel/item/category":true,
+	"/rss/channel/item/enclosure":true,
+	"/rss/channel/item/media:content":true,
+	"/rss/channel/item/media:group":true,
+	"/rss/channel/item/media:group/media:content":true,
+}
+
+jxml.xmap.atom={
+	"/feed/category":true,
+	"/feed/link":true,
+	"/feed/entry":true,
+	"/feed/entry/category":true,
+	"/feed/entry/link":true,
+}
+
+
+
+jxml.sanistr=function(s)
+{
+	s=""+s
+	const map = {
+		'&': '&',
+		'<': '&lt;',
+		'>': '&gt;',
+		'"': '&quot;',
+	};
+	return s.replace(/[&<>"]/ig, function(match){return map[match]});
+}
+
+/*
+
+Create a json object from an xml string using the given xmap.
+
+The xmap is a dictionary of xpath:true for each element that may occur 
+multiple times.
+
+Not providing an xmap will make us assume that every element can occur 
+multiple times which always work but will create an ugly/inefficient 
+json format.
+
+*/
+jxml.parse_xml=function(data,xmap)
+{
+	let json=[];
+	let stack=[];
+	let top={};stack.push(top);
+	let ret=top
+	let xpath=[""]
+	let opath=[""]
+	let cdata=false;
+
+	let parser = sax.parser(true)
+	
+	let ismulti=function(path)
+	{
+		if(!xmap) { return true } // if no map then everything is multi
+		if("function"===typeof xmap) { return xmap(path) }// only approved paths are multi
+		if(xmap[path]) { return true } // only given paths are multi
+		return false
+	}
+
+	parser.onopentag=function(node){
+		let tag="/"+node.name.toLowerCase()
+
+		let path=xpath.join("")+tag
+		let tagpath=opath.at(-1)+tag
+		if( ismulti(path) )
+		{
+			let parent=top
+			top={}
+			if(! parent[tagpath] ) { parent[tagpath]=[] }
+			parent[tagpath].push(top)
+			for(let n in node.attributes) { top["@"+n.toLowerCase()]=node.attributes[n] }
+			stack.push(top)
+			xpath.push(tag)
+			opath.push("")
+		}
+		else
+		{
+			for(let n in node.attributes) { top[tagpath+"@"+n.toLowerCase()]=node.attributes[n] }
+			stack.push(top)
+			xpath.push(tag)
+			opath.push(tagpath)
+		}
+//		console.log("+"+node.name+" "+path)
+	}
+
+	parser.onclosetag=function(name){
+		stack.pop()
+		top=stack.at(-1)
+		xpath.pop()
+		opath.pop()
+//		console.log("-"+name)
+	}
+
+	parser.ontext=function(text){
+		text=text.trim()
+		if(text!="") // ignore white space
+		{
+			let tag=opath.at(-1)
+			if( top[tag] )
+			{
+				top[tag]=top[tag]+" "+text
+			}
+			else
+			{
+				top[tag]=text
+			}
+		}
+	}
+	parser.oncdata=parser.ontext // oncdata is same function as ontext
+
+// maintain cdata text flag
+	parser.onopencdata=function(){ cdata=true; }
+	parser.onclosecdata=function(){ cdata=false; }
+
+//throw any errors
+	parser.onerror=function(e){ throw new Error(e) }
+
+	parser.write(data);
+
+	if(stack.length!=1)
+	{
+		throw new Error("premature ending in xml data")
+	}
+
+	return ret
+}
+
+// expand a json xml object so each element is forced into an array
+// this guarantees the format is what you would have gotten from a parse without an xmap
+jxml.expand=function(it,paths)
+{
+	let ret=jxml.expand_one(it)
+	jxml.recurse(ret,jxml.expand,paths)
+	return ret
+}
+// perform top level expansion only, still need to recurse
+jxml.expand_one=function(it)
+{
+	let ret={}
+	for(let path in it ) // make arrays
+	{
+		if( (path=="") || (path[0]=="@") ) // simple
+		{
+			ret[path]=it[path] // this should never be an array
+		}
+		else // find split and put in an array
+		{
+			let idx=path.indexOf("/",1) // skip first char which can be a / or @
+			let idx2=path.indexOf("@",1) // skip first char which can be a / or @
+			if( (idx2>0) && ( (idx2<idx) || (idx<0) ) ) { idx=idx2 } // second / or first @
+			if( (idx<0) && Array.isArray(it[path]) ) // full path, no / or @ to split and already an array
+			{
+				ret[path]=[ ...it[path] ] // copy array
+			}
+			else
+			{
+				if( idx<0 ) // a text node
+				{
+					idx=path.length
+				}
+				let base=path.substring(0,idx)
+				if( !Array.isArray(ret[base]) ) { ret[base]=[{}] } // make array with one object
+				ret[base][0][ path.substring(idx) ]=it[path] // copy into first object ( we will dupe the array on next pass )
+			}
+		}
+	}
+	return ret
+}
+
+// call recursively into arrays with an array of path strings
+jxml.recurse=function(it,func,paths)
+{
+	paths=paths||[]
+	for(let path in it ) // do arrays
+	{
+		if(Array.isArray(it[path]))
+		{
+			paths.push(path)
+			for(let i=0;i<it[path].length;i++)
+			{
+				it[path][i]=func(it[path][i],paths)
+			}
+			paths.pop()
+		}
+	}
+	return it
+}
+
+jxml.build_xml=function(data)
+{
+	let ss=['<?xml version="1.0" encoding="UTF-8" ?>\n']
+
+	data=jxml.expand(data) // fully expand so every sub tag is an array
+	
+	let parse
+	parse=function(it,paths)
+	{
+		let top=paths[paths.length-1] // this will start with a /
+		let indent=(" ").repeat(Math.max(0,paths.length-1))
+		let children=false
+		if(top) // catch the starting empty paths
+		{
+			ss.push(indent+"<"+top.substring(1))
+			for(let path in it )
+			{
+				if(path[0]=="@") // add atributes
+				{
+					ss.push(" "+path.substring(1)+"=\""+jxml.sanistr(it[path])+"\"")
+				}
+				if((path[0]=="/")||(path=="")) // children
+				{
+					children=true
+				}
+			}
+			if(children)
+			{
+				ss.push(" >\n")
+				if(it[""]){ss.push(indent+" "+jxml.sanistr(it[""])+"\n")} // push text
+			}
+			else
+			{
+				ss.push(" />\n")
+			}
+		}
+		jxml.recurse(it,parse,paths) // add sub tags
+		if(top)
+		{
+			if(children)
+			{
+				ss.push(indent+"<"+top+">\n")
+			}
+		}
+	}
+	parse(data,[])
+
+	
+	let s=ss.join("")
+	return s
+}

package/npm-publish

@@ -0,0 +1,11 @@
+
+cd `dirname $0`
+
+git add .
+git commit -m.
+npm version patch
+git add .
+git commit -m.
+git push
+npm publish
+

package/package.json

@@ -0,0 +1,35 @@
+{
+	"name": "@xriss/weerss",
+	"version": "0.0.2",
+	"source": "weerss.js",
+	"type": "module",
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/xriss/weerss.git"
+	},
+	"bin": {
+		"weerss": "./cmd.import.js"
+	},
+	"scripts": {
+		"watch": " webpack watch --config webpack.config.cjs --mode development ",
+		"build": " webpack build --config webpack.config.cjs --mode production "
+	},
+	"dependencies": {
+		"@xriss/djon": "^0.0.14",
+		"express": "^4.19.2",
+		"minimist": "^1.2.8",
+		"move-file": "^3.1.0",
+		"parse-torrent": "^11.0.17",
+		"path-browserify": "^1.0.1",
+		"promised-sqlite3": "^2.1.0",
+		"safe-stable-stringify": "^2.5.0",
+		"sanitize-filename": "^1.6.3",
+		"sax": "^1.4.1",
+		"sqlite3": "^5.1.7",
+		"stream-browserify": "^3.0.0",
+		"webpack": "^5.94.0"
+	},
+	"devDependencies": {
+		"webpack-cli": "^5.1.4"
+	}
+}

package/register-hooks.js

@@ -0,0 +1,35 @@
+
+// default to just .txt
+let extensions = /\.(txt)$/i;
+
+// copy raw-loader pattern so we are in sync
+import wp from "./webpack.config.cjs"
+for( let it of wp && wp.module && wp.module.rules || [] )
+{
+	if(it.use=="raw-loader")
+	{
+		extensions=it.test
+		break
+	}
+}
+
+// hook into imports
+export async function load(url, context, nextLoad)
+{
+	if (extensions.test(url))
+	{
+		let { source: source } = await nextLoad(url, { ...context , format:"module" } );
+		let data=JSON.stringify(source.toString())
+		source=`
+const text=${data};
+export default text;
+`
+		return {
+			format: "module",
+			shortCircuit: true,
+			source: source
+		};
+	}
+	return nextLoad(url);
+}
+

package/register.js

@@ -0,0 +1,3 @@
+// just register hooks
+import module from "module"
+await module.register("./register-hooks.js",import.meta.url)

package/serv.js

@@ -0,0 +1,21 @@
+
+const serv={}
+export default serv
+
+import express from 'express'
+
+serv.start=function()
+{
+
+    const app = express()
+    const port = 3000
+
+    app.get('/', (req, res) => {
+        res.send('Hello World!')
+    })
+
+    app.listen(port, () => {
+        console.log(`Example app listening on port ${port}`)
+    })
+
+}