@xriss/weerss 0.0.2

package/cmd.import.js

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import('@xriss/weerss/cmd.js').catch((err) => { console.error(err); process.exit(1) })

package/cmd.js

@@ -0,0 +1,106 @@
+
+const cmd={}
+export default cmd
+
+
+import minimist from "minimist"
+
+import path from "path"
+import url from "url"
+
+import serv from "./serv.js"
+import db from "./db.js"
+
+import hoard from "./hoard.js"
+import feeds from "./feeds.js"
+import items from "./items.js"
+
+import weerss from "./weerss.js"
+
+const args = minimist(process.argv.slice(2),{boolean:true})
+
+args.config=args.config || process.env.WEERSS_CONFIG
+
+await weerss.load_config(args)
+
+let arg1=args._[0] || "help"
+let arg2=args._[1]
+
+if( arg1=="help" )
+{
+	console.log(`
+
+options:
+
+	Load config file, you should always load your custom config for all 
+	of the following commands...
+
+		--config=weerss.config
+
+	environment variables can also set options with the prefix WEERSS_ 
+	and the option name in all uppercase. eg:
+
+		export WEERSS_CONFIG=/home/dave/weerss.config
+
+commands:
+
+tvrss config
+	Print current config.
+
+tvrss config FILENAME
+	Save current (default?) config to FILENAME.
+
+tvrss fetch
+	Fetch all feeds and scan files with tvmaze.
+
+tvrss list
+	List torrents in db.
+
+tvrss rss
+	Save torrents as rss.
+
+tvrss move
+	Move downloaded chaotic files to an organised location.
+
+tvrss dirs
+	Create a script to cleanup TV dirs using tvmaze.
+
+tvrss help
+	Print this help message.
+
+`)
+}
+else
+if( arg1=="config" )
+{
+	await weerss.save_config(args)
+}
+else
+if( arg1=="fetch" )
+{
+	await weerss.fetch(args)
+}
+else
+if( arg1=="list" )
+{
+	await weerss.list(args)
+}
+else
+if( arg1=="rss" )
+{
+	await weerss.save_rss(args)
+}
+else
+if( arg1=="move" )
+{
+	await weerss.move(args)
+}
+else
+if( arg1=="dirs" )
+{
+	await weerss.dirs(args)
+}
+else
+{
+	console.log(` Unknown weerss command "${arg1}" `)
+}

package/db.js

@@ -0,0 +1,245 @@
+
+let db={}
+export default db
+
+
+import { AsyncDatabase } from "promised-sqlite3"
+
+import { configure } from 'safe-stable-stringify'
+const stringify = configure({})
+
+import weerss from "./weerss.js"
+
+
+db.test=async function()
+{
+	await db.setup()
+	await db.set(null,"hello","world")
+	await db.set(null,"hell","{world:poop}")
+	
+	console.log( await db.list() )
+	
+	await db.close()
+}
+
+
+
+db.name="weerss"
+
+db.setup=async function()
+{
+
+	db.handle = await AsyncDatabase.open( weerss.get_config_path("sqlite_filename") );
+
+//	db.handle.inner.on("trace", (sql) => console.log("[TRACE]", sql));
+
+	await db.handle.run("PRAGMA synchronous = 0 ;");
+	await db.handle.run("PRAGMA encoding = \"UTF-8\" ;");
+	await db.handle.run("PRAGMA journal_mode=WAL;");
+	await db.handle.run("PRAGMA mmap_size=268435456;");
+	await db.handle.run("PRAGMA temp_store=2;");
+
+
+	await db.handle.run(`
+ 
+		CREATE TABLE IF NOT EXISTS keyval
+		(
+			key STRING PRIMARY KEY ,
+			value JSONB NOT NULL
+		);
+ 
+    `);
+
+	await db.handle.run(`
+ 
+		CREATE TABLE IF NOT EXISTS hoard
+		(
+			key STRING PRIMARY KEY ,
+			value JSONB NOT NULL
+		);
+ 
+    `);
+
+	await db.handle.run(`
+ 
+		CREATE TABLE IF NOT EXISTS feeds
+		(
+			key STRING PRIMARY KEY ,
+			value JSONB NOT NULL
+		);
+ 
+    `);
+
+	await db.handle.run(`
+ 
+		CREATE TABLE IF NOT EXISTS items
+		(
+			key STRING PRIMARY KEY ,
+			value JSONB NOT NULL
+		);
+ 
+    `);
+
+	await db.handle.run(`
+ 
+		CREATE TABLE IF NOT EXISTS torrents
+		(
+			key STRING PRIMARY KEY ,
+			value JSONB NOT NULL
+		);
+ 
+    `);
+
+	await db.handle.run(`
+ 
+		CREATE TABLE IF NOT EXISTS shows
+		(
+			key STRING PRIMARY KEY ,
+			value JSONB NOT NULL
+		);
+ 
+    `);
+
+}
+
+db.close=async function()
+{
+	await db.handle.close()
+	db.handle=null
+}
+
+db.get=async function(table,key)
+{
+	table=table||"keyval"
+	let row=await db.handle.get(`
+
+		SELECT * FROM ${table}
+		WHERE key=$key;
+
+	`,{$key:key})
+	return row && row.value && JSON.parse(row.value)
+}
+
+db.set=async function(table,key,value)
+{
+	table=table||"keyval"
+	await db.handle.run(`
+
+		INSERT INTO ${table} (key,value)
+		VALUES ($key,$value)
+		ON CONFLICT (key)
+		DO 
+			UPDATE SET value = $value
+			WHERE key = $key;
+
+	`,{$key:key,$value:stringify(value)})
+}
+
+
+db.clear=async function(table)
+{
+	table=table||"keyval"
+	await db.handle.run(`
+
+		DELETE FROM ${table};
+
+	`,{})
+}
+
+db.delete=async function(table,key)
+{
+	table=table||"keyval"
+	await db.handle.run(`
+
+		DELETE FROM ${table} WHERE key=$key;
+
+	`,{$key:key})
+}
+
+db.list=async function(table,filter)
+{
+	table=table||"keyval"
+	filter=filter || {}
+	let rs=[]
+	
+	let filters=[]
+	for(let fb in filter)
+	{
+		let fv=filter[fb]
+		if(typeof fv=="Number")
+		{
+			fv=String(fv)
+		}
+		else
+		{
+			fv='"'+String(fv)+'"'
+		}
+
+		if(fb.endsWith("_is_null"))
+		{
+			let f=fb.substring(0,fb.length-8)
+			filters.push(" json_extract (value, '$."+f+"') IS NULL ")
+		}
+		else
+		if(fb.endsWith("_is_not_null"))
+		{
+			let f=fb.substring(0,fb.length-12)
+			filters.push(" json_extract (value, '$."+f+"') IS NOT NULL ")
+		}
+		else
+		if(fb.endsWith("_lt"))
+		{
+			let f=fb.substring(0,fb.length-3)
+			filters.push(" json_extract (value, '$."+f+"') < "+fv)
+		}
+		else
+		if(fb.endsWith("_lteq"))
+		{
+			let f=fb.substring(0,fb.length-5)
+			filters.push(" json_extract (value, '$."+f+"') <= "+fv)
+		}
+		else
+		if(fb.endsWith("_gt"))
+		{
+			let f=fb.substring(0,fb.length-3)
+			filters.push(" json_extract (value, '$."+f+"') > "+fv)
+		}
+		else
+		if(fb.endsWith("_gteq"))
+		{
+			let f=fb.substring(0,fb.length-5)
+			filters.push(" json_extract (value, '$."+f+"') >= "+fv)
+		}
+		else
+		if(fb.endsWith("_not"))
+		{
+			let f=fb.substring(0,fb.length-4)
+			filters.push(" json_extract (value, '$."+f+"') != "+fv)
+		}
+		else
+		{
+			filters.push(" json_extract (value, '$."+fb+"') = "+fv)
+		}
+	}
+	if( filters.length==0 )
+	{
+		filters=""
+	}
+	else
+	{
+		filters="WHERE "+filters.join(" AND\n")
+	}
+//	console.log(filters)
+	
+	await db.handle.each(`
+	
+		SELECT * FROM ${table}
+		${filters}
+
+	`,{},function(row)
+	{
+		rs.push(JSON.parse(row.value))
+	})
+
+	return rs // filtered only, sorted by date
+}

package/feeds.js

@@ -0,0 +1,321 @@
+
+let feeds={}
+export default feeds
+
+
+import      items     from "./items.js"
+import      db        from "./db.js"
+import      jxml      from "./jxml.js"
+import      hoard     from "./hoard.js"
+
+
+import { configure } from 'safe-stable-stringify'
+const stringify = configure({})
+
+
+feeds.cached={}
+
+feeds.precache=async function() // faster then individual cache, db is fecking slow
+{
+	for(let feed of await db.list("feeds"))
+	{
+		feeds.cache(feed.url,feed)
+	}
+}
+
+feeds.cache=async function(url,feed) // probably fast
+{
+	if(feed) { feeds.cached[url]=feed ; return feed }
+	if(feeds.cached[url]) { return feeds.cached[url] }
+	await feeds.get(url)
+	return feeds.cached[url]
+}
+feeds.get=async function(url) // always slow
+{
+	let feed=await db.get("feeds",url)
+	if(feed) { feeds.cache(feed.url,feed) }
+	return feed
+}
+feeds.set=async function(feed)
+{
+	await db.set("feeds",feed.url,feed)
+	if(feed) { feeds.cache(feed.url,feed) }
+}
+
+feeds.prepare=function(feed)
+{
+	let atom=feed.atom||{}
+	let rss=feed.rss||{}
+	
+	if(rss["/rss/channel/title"])
+	{
+		feed.title=rss["/rss/channel/title"]
+	}
+	else
+	if(atom["/feed/title"])
+	{
+		feed.title=atom["/feed/title"]
+	}
+
+	if(rss["/rss/channel/pubdate"])
+	{
+		feed.date=new Date(rss["/rss/channel/pubdate"])
+	}
+	else
+	if(atom["/feed/updated"])
+	{
+		feed.date=new Date(atom["/feed/updated"])
+	}
+
+	return feed
+}
+
+feeds.add=async function(it)
+{
+	let old=await feeds.cache(it.url) // await db.get("feeds",it.url)
+
+	let feed={} // merge old and new here
+	if(old)
+	{
+		for(let n in old ){ feed[n]=old[n] }
+	}
+	for(let n in it ){ feed[n]=it[n] }
+	// maintain defaults
+	feed.date=feed.date || new Date()
+	feed.fails=feed.fails || 0
+	
+	if( stringify(old) != stringify(feed) ) // something changed
+	{
+		await feeds.set(feed)
+	}
+}
+
+
+feeds.add_opml=async function(data,url)
+{
+	if(url){data=await hoard.fetch_text(url,true)}
+	if(!data){return}
+	let jsn=jxml.parse_xml(data,jxml.xmap.opml)
+//console.log(jsn)
+
+	let check;check=async function(it)
+	{
+		if( it["@xmlurl"] )
+		{
+			let feed={}
+			feed.url=it["@xmlurl"]
+			await feeds.add(feed)
+		}
+		if(it["/outline"]) // check sub outlines
+		{
+			for(let it2 of it["/outline"] )
+			{
+				await check(it2)
+			}
+		}
+	}
+
+	if( jsn["/opml/body/outline"] )
+	{
+		for(let it of jsn["/opml/body/outline"] )
+		{
+			await check(it)
+		}
+	}
+	else
+	if( jsn["/rss/channel/item/title"] ) // looks like a rss
+	{
+		if(url) // so subscribe to this url
+		{
+			let feed={}
+			feed.url=url
+			await feeds.add(feed)
+		}
+	}
+	else
+	if( jsn["/feed/entry/title"] ) // looks like attom
+	{
+		if(url) // so subscribe to this url
+		{
+			let feed={}
+			feed.url=url
+			await feeds.add(feed)
+		}
+	}
+}
+
+feeds.build_opml=async function()
+{
+	let j={
+ "/opml/head/title": "RERSS filter",
+ "/opml@version": "1.0",
+ "/opml/body/outline": [],
+}
+
+	let out=j["/opml/body/outline"]
+	
+	let feeds=await db.list("feeds")
+	let now=(new Date()).getTime()
+	
+	for(let feed of feeds)
+	{
+		if(!feed.off) // disabled feeds are not exported
+		{
+			let it={}
+			it["@type"]="rss"
+			it["@xmlUrl"]=feed.url
+			it["@text"]=feed.title
+			it["@title"]=feed.title
+			out.push(it)
+		}
+	}
+	let x=jxml.build_xml(j)
+	return x
+}
+
+feeds.load_opml=async function()
+{
+/*
+	let data=await gist.read("weerss.opml")
+	await feeds.add_opml(data)
+*/
+}
+
+feeds.save_opml=async function()
+{
+/*
+	let data=await feeds.build_opml()
+	await gist.write("weerss.opml",data)
+*/
+}
+
+feeds.list_length=0
+feeds.list_length_count=0
+feeds.fetch_all=async function()
+{
+	console.log("starting feeds fetch_all")
+
+	let rets=[]
+	let list=await db.list("feeds")
+		
+	feeds.list_length=list.length
+	feeds.list_length_count=0
+	
+	// try and get popular feeds first
+	list.sort(function(a,b){
+		return ( b.items_date || 0 ) - ( a.items_date || 0 )
+	})
+	
+	let topitem=(await db.list("items",{},"date","prev",1))[0] ; topitem=topitem && topitem.uuid
+
+	for(let feed of list )
+	{
+		await feeds.cache(feed.url,feed)
+		await feeds.fetch(feed)
+	}
+
+	if(items.add_count>0)
+	{
+		console.log("+"+items.add_count)
+	}
+	else
+	{
+		console.log("")
+	}
+
+}
+
+feeds.fetch=async function(feed)
+{
+	console.log(feeds.list_length_count +"/"+ feeds.list_length + " +"+items.add_count)
+	if(!feed.off) // check if the feed is turned off
+	{
+	try{
+		
+		console.log("fetching feed",feed.url)
+		let before_count=items.add_count
+
+		
+		let txt=await hoard.fetch_text(feed.url)
+
+		let e1,e2
+		try{ feed.rss=jxml.parse_xml(txt,jxml.xmap.rss) }
+		catch(e){e1=e}
+
+		try{ feed.atom=jxml.parse_xml(txt,jxml.xmap.atom) }
+		catch(e){e2=e}
+		
+		if( e1 && e2 ) // something really went wrong
+		{
+			console.log(e1)
+			console.log(e2)
+		}
+
+		feeds.prepare(feed)
+		
+		// feed.items_date date is date of most recent item we have seen in the feed
+		let check_date=function(it)
+		{
+			if(it.date)
+			{
+				if(feed.items_date)
+				{
+					if( feed.items_date < it.date )
+					{
+						feed.items_date=it.date
+					}
+				}
+				else
+				{
+					feed.items_date=it.date
+				}
+			}
+		}
+
+		if(feed.rss && feed.rss["/rss/channel/item"])
+		{
+			feed.items_count=0
+			for(let rss_item of feed.rss["/rss/channel/item"] )
+			{
+				let item={rss:rss_item}
+				feed.items_count++
+				items.prepare(item,feed)
+				check_date(item)
+				await items.add(item)
+//				await hoard.first_text(item.link)
+			}
+			feed.fails=0
+		}
+		else
+		if(feed.atom && feed.atom["/feed/entry"])
+		{
+			feed.items_count=0
+			for(let atom_entry of feed.atom["/feed/entry"] )
+			{
+				let item={atom:atom_entry}
+				feed.items_count++
+				items.prepare(item,feed)
+				check_date(item)
+				await items.add(item)
+//				await hoard.first_text(item.link)
+			}
+			feed.fails=0
+		}
+		else
+		{
+			feed.fails=(feed.fails||0)+1
+		}
+
+		await db.set("feeds",feed.url,feed) // save updated feed info
+
+		let count=items.add_count-before_count
+		if(count>0)
+		{
+			console.log("added "+count+" new items")
+		}
+	}catch(e){console.error(e)}
+	}
+
+	feeds.list_length_count++
+}
+