litestack 0.2.3 → 0.3.0

data/lib/litestack/litemetric.sql.yml

@@ -2,68 +2,338 @@ schema:
   1:
     create_topics: >
       CREATE TABLE IF NOT EXISTS topics(
-        name text PRIMARY KEY NOT NULL
-      ) WITHOUT ROWID; 
+        name TEXT PRIMARY KEY NOT NULL,
+        state TEXT,
+        updated_at INTEGER
+      ); 
+    
     create_events: >
       CREATE TABLE IF NOT EXISTS events(
-        topic text NOT NULL references topics(name) ON DELETE CASCADE, 
-        name TEXT NOT NULL, 
-        key TEXT NOT NULL, 
+        topic TEXT NOT NULL references topics(name) ON DELETE CASCADE, 
+        name TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE, 
+        key TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE, 
         count INTEGER DEFAULT(0) NOT NULL ON CONFLICT REPLACE, 
-        value INTEGER, 
-        minimum INTEGER, 
-        maximum INTEGER, 
+        value REAL, 
+        minimum REAL, 
+        maximum REAL, 
         created_at INTEGER DEFAULT((unixepoch()/300*300)) NOT NULL, 
         resolution TEXT DEFAULT('minute') NOT NULL, 
-        PRIMARY KEY(resolution, topic, name, key, created_at)
-      ) WITHOUT ROWID;    
-    create_index_on_event: CREATE INDEX IF NOT EXISTS events_by_resolution ON events(resolution, created_at);
+        PRIMARY KEY(resolution, created_at, topic, name, key)
+      ) STRICT;        
+
+    create_topic_index_on_events: CREATE INDEX events_topic_index ON events (resolution, created_at, topic) WHERE name = '___';
+    create_event_index_on_events: CREATE INDEX events_event_index ON events (resolution, created_at , topic, name) WHERE key = '___';
+
+    create_events_search: >
+      CREATE VIRTUAL TABLE IF NOT EXISTS events_search USING fts5 (
+        topic, 
+        name, 
+        key, 
+        resolution, 
+        content=events, 
+        tokeniz="trigram"
+      )
+    
+    create_events_insert_trigger: >
+      CREATE TRIGGER events_insert AFTER INSERT ON events 
+      BEGIN
+        INSERT INTO 
+          events_search(rowid, topic, name, key, resolution) 
+        VALUES
+          (new.rowid, new.topic, new.name, new.key, new.resolution);
+      END;
+    
+    create_events_delete_trigger: >
+      CREATE TRIGGER events_delete AFTER DELETE ON events 
+      BEGIN
+        INSERT INTO
+          events_search(events_search, rowid, topic, name, key, resolution) 
+        VALUES
+          ('delete', old.rowid, old.topic, old.name, old.key, old.resolution);        
+      END;
+
+    create_events_update_trigger: >
+      CREATE TRIGGER events_update AFTER UPDATE ON events 
+      BEGIN
+        INSERT INTO
+          events_search(events_search, rowid, topic, name, key, resolution) 
+        VALUES
+          ('delete', old.rowid, old.topic, old.name, old.key, old.resolution);        
+        INSERT INTO 
+          events_search(rowid, topic, name, key, resolution) 
+        VALUES
+          (new.rowid, new.topic, new.name, new.key, new.resolution);
+      END;
+
+    rebuild_index: >
+      INSERT INTO events_search(events_search) VALUES ('rebuild');
+    
+       
+    
 
 stmts:
-  # register topic
-  register_topic: INSERT INTO topics VALUES (?) ON CONFLICT DO NOTHING;
+  
+  register_topic: >
+    INSERT INTO topics (name, updated_at) VALUES (?, unixepoch()) 
+    ON CONFLICT DO
+    UPDATE SET updated_at = unixepoch();
+    
+  capture_state: >
+    UPDATE topics 
+    SET state = ?2, updated_at = unixepoch() 
+    WHERE name = ?1
+
+  snapshot: >
+    SELECT state, datetime(updated_at, 'unixepoch')
+    FROM topics 
+    WHERE name = ?1
 
   capture_event: >
-    INSERT INTO events(topic, name, key, created_at, count, value, minimum, maximum) VALUES ($1, $2, $3, $4, $5, $6, $6, $6) 
+    INSERT INTO events(topic, name, key, created_at, count, value, minimum, maximum) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) 
     ON CONFLICT DO 
     UPDATE SET count = count + EXCLUDED.count, value = value + EXCLUDED.value, minimum = min(minimum, EXCLUDED.minimum), maximum = max(maximum, EXCLUDED.maximum) 
   
   # requires an index on (resolution, created_at)
   summarize_events: >
-    INSERT INTO events (topic, name, key, count, value, minimum, maximum, created_at, resolution ) SELECT 
+    INSERT INTO 
+      events (topic, name, key, count, value, minimum, maximum, created_at, resolution ) 
+    SELECT 
       topic, 
       name, 
       key, 
-      sum(count) as count, 
-      sum(value) as value, 
-      min(minimum) as minimum, 
-      max(maximum) as maximum, 
-      (created_at/$1)*$1 as created,  
-      $2 
-    FROM events WHERE resolution = $3 AND created_at < (unixepoch()/$1)*$1 GROUP BY topic, name, key, created ON CONFLICT DO UPDATE 
-    SET count = count + EXCLUDED.count, value = value + EXCLUDED.value, minimum = min(minimum, EXCLUDED.minimum), maximum = max(maximum, EXCLUDED.maximum); 
+      sum(count), 
+      sum(value), 
+      min(minimum), 
+      max(maximum), 
+      (created_at/(?1))*?1 as created,  
+      ?2 
+    FROM events 
+    WHERE resolution = ?3 
+    GROUP BY topic, name, key, created
+    ON CONFLICT DO
+    UPDATE SET count = EXCLUDED.count, value = EXCLUDED.value, minimum = EXCLUDED.minimum, maximum = EXCLUDED.maximum 
 
-  # requires an index on (resolution, created_at)
-  delete_events: DELETE FROM events WHERE resolution = $3 AND created_at < (unixepoch() - $4);
+  # requires an index on (resolution, created_at) #TODO: fix deletion boundary (use the date mapping)
+  delete_events: DELETE FROM events WHERE resolution = ?1 AND (created_at /(?2)) * ?2  < ((unixepoch()/(?2)) * ?2 - ?2);
 
   # select topics from the topics table
-  list_topics: SELECT name FROM topics;
-
-  # requires an index on (resolution, topic, name)
-  list_event_names: >
-    SELECT name, sum(count) as count, count(distinct name) as name, sum(value) as value, min(minimum), max(maximum) 
-    FROM events WHERE resolution = ? AND topic = ? GROUP BY name ORDER BY count;
+  list_topics: SELECT name FROM topics ORDER BY name ASC;
 
-  # requires an index on (resolution, topic, name, key)
-  list_event_keys: >
-    SELECT key, sum(count) as count, sum(value) as value, min(minimum), max(maximum)
-    FROM events WHERE resolution = ? AND topic = ? AND name = ? GROUP BY key ORDER BY count;
+  # select topics, their event counts, total values, min and max values 
+  topics_summaries: >
+    SELECT 
+      topic, 
+      count(distinct name) as events, 
+      count(distinct key) as keys, 
+      sum(count) as rcount 
+    from events 
+    where resolution = ?1 
+    AND created_at <= unixepoch()
+    AND created_at >= unixepoch() -?2 
+    AND name = '___'
+    AND iif(length(?4) > 0, rowid IN (SELECT rowid FROM events_search WHERE topic LIKE ?4), TRUE)
+    GROUP BY topic 
+    ORDER BY 
+      iif(?3 = 'events', events, 
+        iif(?3 = 'keys', keys, 
+          iif(?3 = 'rcount', rcount, topic)))  
+    DESC;
   
-  # requires an index on (resolution, topic, name, key, created_at)
-  list_events_by_key: > 
-    SELECT * FROM events WHERE resolution = $1 AND topic = $2 AND name = $3 AND key = $4 ORDER BY created_at ASC;    
-    
-  # requires an index on (resolution, topic, name, key, created_at)
-  list_all_events: >
-    SELECT * FROM events WHERE resolution = ? AND topic = ? ORDER BY name, key, created_at ASC;
+  topics_summaries_asc: >
+    SELECT 
+      topic, 
+      count(distinct name) as events, 
+      count(distinct key) as keys, 
+      sum(count) as rcount 
+    from events 
+    where resolution = ?1 
+    AND created_at <= unixepoch()
+    AND created_at >= unixepoch() -?2 
+    AND name = '___'
+    AND iif(length(?4) > 0, rowid IN (SELECT rowid FROM events_search WHERE topic LIKE ?4), TRUE)
+    GROUP BY topic 
+    ORDER BY 
+      iif(?3 = 'events', events, 
+        iif(?3 = 'keys', keys, 
+          iif(?3 = 'rcount', rcount, topic)))  
+    ASC;
+
+  events_summaries: >
+    SELECT 
+      name, 
+      count(DISTINCT key) AS keys, 
+      sum(count) as rcount,
+      cast(sum(value) as double) / sum(count) as ravg, 
+      sum(value) AS rtotal,       
+      min(minimum) AS rmin, 
+      max(maximum) AS rmax 
+    FROM events 
+    WHERE topic = ?1 AND resolution = ?2 
+    AND created_at <= unixepoch()
+    AND created_at >= unixepoch() - ?5 
+    AND name != '___'
+    AND key = '___'
+    AND iif(length(?4) > 0, rowid IN (SELECT rowid FROM events_search WHERE name LIKE ?4), TRUE)
+    GROUP BY name 
+    ORDER BY 
+      iif(?3 = 'rcount', rcount, 
+        iif(?3 = 'ravg', ravg, 
+          iif(?3 = 'rtotal', rtotal,  
+            iif(?3 = 'rmin', rmin, 
+              iif(?3 = 'rmax', rmax, 
+                iif(?3 = 'keys', keys, 
+                  iif(?3 = 'name', name, rcount))))))) 
+    DESC
+    LIMIT 20;
+
+  events_summaries_asc: >
+    SELECT 
+      name, 
+      count(DISTINCT key) AS keys, 
+      sum(count) as rcount,
+      cast(sum(value) as double) / sum(count) as ravg, 
+      sum(value) AS rtotal,       
+      min(minimum) AS rmin, 
+      max(maximum) AS rmax 
+    FROM events 
+    WHERE topic = ?1 AND resolution = ?2 
+    AND created_at <= unixepoch()
+    AND created_at >= unixepoch() -?5 
+    AND name != '___'
+    AND key = '___'
+    AND iif(length(?4) > 0, rowid IN (SELECT rowid FROM events_search WHERE name LIKE ?4), TRUE)
+    GROUP BY name 
+    ORDER BY 
+      iif(?3 = 'rcount', rcount, 
+        iif(?3 = 'ravg', ravg, 
+          iif(?3 = 'rtotal', rtotal,  
+            iif(?3 = 'rmin', rmin, 
+              iif(?3 = 'rmax', rmax, 
+                iif(?3 = 'keys', keys, 
+                  iif(?3 = 'name', name, rcount))))))) 
+    ASC
+    LIMIT 20;
+
  
+  keys_summaries: >
+    SELECT 
+      key, 
+      sum(count) AS rcount,
+      cast(sum(value) AS double) / sum(count) AS ravg, 
+      sum(value) AS rtotal,       
+      min(minimum) AS rmin, 
+      max(maximum) AS rmax 
+    FROM events 
+    WHERE topic = ?1 AND name = ?2 AND resolution = ?3 
+    AND created_at <= unixepoch()
+    AND created_at >= unixepoch() - ?6 
+    AND key != '___'
+    AND iif(length(?5) > 0, rowid IN (SELECT rowid FROM events_search WHERE key LIKE ?5), TRUE)
+    GROUP BY key
+    ORDER BY 
+      iif(?4 = 'rcount', rcount, 
+        iif(?4 = 'ravg', ravg, 
+          iif(?4 = 'rtotal', rtotal,  
+            iif(?4 = 'rmin', rmin, 
+              iif(?4 = 'rmax', rmax, 
+                iif(?4 = 'key', key, rcount)))))) 
+      DESC
+    LIMIT 20;
+
+  keys_summaries_asc: >
+    SELECT 
+      key, 
+      sum(count) AS rcount,
+      cast(sum(value) AS double) / sum(count) AS ravg, 
+      sum(value) AS rtotal,       
+      min(minimum) AS rmin, 
+      max(maximum) AS rmax 
+    FROM events 
+    WHERE topic = ?1 AND name = ?2 AND resolution = ?3 
+    AND created_at <= unixepoch()
+    AND created_at >= unixepoch() - ?6 
+    AND key != '___'
+    AND iif(length(?5) > 0, rowid IN (SELECT rowid FROM events_search WHERE key LIKE ?5), TRUE)
+    GROUP BY key
+    ORDER BY 
+      iif(?4 = 'rcount', rcount, 
+        iif(?4 = 'ravg', ravg, 
+          iif(?4 = 'rtotal', rtotal,  
+            iif(?4 = 'rmin', rmin, 
+              iif(?4 = 'rmax', rmax, 
+                iif(?4 = 'key', key, rcount)))))) 
+      ASC
+    LIMIT 20;
+
+  # list data points at the topic, event, or key level 
+  # needs: duration, limit, resolution, topic, name (optional), key (optional) 
+
+  topic_data_points: >
+    SELECT 
+      datetime(ts.slot, 'unixepoch') AS rtime, 
+      ts.slot AS unixtime, 
+      sum(events.count) AS rcount, 
+      avg(events.value/events.count) AS ravg 
+    FROM (
+      WITH RECURSIVE timeslot(x) AS ( 
+        SELECT (unixepoch()/(?1))*(?1) 
+        UNION ALL 
+        SELECT x - (?1) FROM timeslot LIMIT ?2 
+      ) SELECT x as slot FROM timeslot 
+    ) AS ts 
+    LEFT OUTER JOIN events 
+    ON ts.slot = events.created_at 
+    AND events.resolution = ?3
+    AND events.topic = ?4 
+    AND events.created_at <= UNIXEPOCH()
+    AND events.name = '___'
+    GROUP BY ts.slot
+    ORDER BY ts.slot ASC;
+    
+  event_data_points: >
+    SELECT 
+      datetime(ts.slot, 'unixepoch') AS rtime, 
+      ts.slot AS unixtime, 
+      events.count AS rcount, 
+      events.value / events.count AS ravg, 
+      events.value AS rtotal 
+    FROM (
+      WITH RECURSIVE timeslot(x) AS ( 
+        SELECT (unixepoch()/(?1))*(?1) 
+        UNION ALL 
+        SELECT x - (?1) FROM timeslot LIMIT ?2 
+      ) SELECT x as slot FROM timeslot 
+    ) AS ts 
+    LEFT OUTER JOIN events 
+    ON ts.slot = events.created_at 
+    AND events.resolution = ?3
+    AND events.topic = ?4 
+    AND events.name = ?5 
+    AND events.key = '___'
+    AND events.created_at <= UNIXEPOCH() 
+    GROUP BY ts.slot
+    ORDER BY ts.slot ASC;
+    
+  key_data_points: >
+    SELECT 
+      datetime(ts.slot, 'unixepoch') AS rtime, 
+      ts.slot AS unixtime, 
+      events.count AS rcount, 
+      events.value / events.count AS ravg, 
+      events.value AS rtotal 
+    FROM (
+      WITH RECURSIVE timeslot(x) AS ( 
+        SELECT (unixepoch()/(?1))*(?1) 
+        UNION ALL 
+        SELECT x - (?1) FROM timeslot LIMIT ?2 
+      ) SELECT x as slot FROM timeslot 
+    ) AS ts 
+    LEFT OUTER JOIN events 
+    ON ts.slot = events.created_at 
+    AND events.resolution = ?3
+    AND events.topic = ?4 
+    AND events.name = ?5 
+    AND events.key = ?6 
+    AND events.created_at <= UNIXEPOCH() 
+    GROUP BY ts.slot
+    ORDER BY ts.slot ASC;

data/lib/litestack/litemetric_collector.sql.yml

@@ -0,0 +1,56 @@
+schema:
+  1:
+    
+    create_events: >
+      CREATE TABLE IF NOT EXISTS local_events(
+        topic TEXT NOT NULL,
+        name TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE, 
+        key TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE, 
+        count INTEGER DEFAULT(0) NOT NULL ON CONFLICT REPLACE, 
+        value REAL, 
+        minimum REAL, 
+        maximum REAL, 
+        created_at INTEGER DEFAULT((unixepoch()/300*300)) NOT NULL ON CONFLICT REPLACE, 
+        resolution TEXT DEFAULT('minute') NOT NULL, 
+        PRIMARY KEY(resolution, created_at, topic, name, key)
+      ) STRICT;        
+
+stmts:
+      
+  capture_event: >
+    INSERT INTO local_events(topic, name, key, created_at, count, value, minimum, maximum) 
+    VALUES 
+      (?1, ?2, ?3, ?4, ?5, ?6, ?6, ?6), 
+      (?1, ?2, '___', ?4, ?5, ?6, ?6, ?6),
+      (?1, '___', '___', ?4, ?5, ?6, ?6, ?6) 
+    ON CONFLICT DO UPDATE 
+    SET 
+      count = count + EXCLUDED.count, 
+      value = value + EXCLUDED.value, 
+      minimum = min(minimum, EXCLUDED.minimum), 
+      maximum = max(maximum, EXCLUDED.maximum) 
+    
+  migrate_events: 
+    INSERT INTO m.events(topic, name, key, created_at, count, value, minimum, maximum) 
+    SELECT topic, name, key, created_at, count, value, minimum, maximum 
+    FROM local_events 
+    ORDER BY resolution, created_at ASC, topic, name, key  
+    LIMIT ?
+    ON CONFLICT DO UPDATE 
+    SET 
+      count = count + EXCLUDED.count, 
+      value = value + EXCLUDED.value, 
+      minimum = min(minimum, EXCLUDED.minimum), 
+      maximum = max(maximum, EXCLUDED.maximum) 
+
+  delete_migrated_events:
+    DELETE FROM local_events WHERE rowid IN (
+      SELECT rowid 
+      FROM local_events 
+      ORDER BY resolution, created_at ASC, topic, name, key 
+      LIMIT ?
+    )
+  
+  event_count:
+    SELECT count(*) FROM local_events;
+   

data/lib/litestack/litequeue.rb

@@ -23,7 +23,7 @@ class Litequeue
   include Litesupport::Liteconnection
   
   DEFAULT_OPTIONS = {
-    path: "./queue.db",
+    path: Litesupport.root.join("queue.sqlite3"),
     mmap_size: 32 * 1024,
     sync: 0
   }
@@ -78,55 +78,54 @@ class Litequeue
   
   # deletes all the entries in all queues, or if a queue name is given, deletes all entries in that specific queue
   def clear(queue=nil)
-    run_sql("DELETE FROM queue WHERE iif(?, name = ?,  1)", queue)
+    run_sql("DELETE FROM queue WHERE iif(?1 IS NOT NULL, name = ?1,  TRUE)", queue)
   end
 
   # returns a count of entries in all queues, or if a queue name is given, reutrns the count of entries in that queue
   def count(queue=nil)
-    run_sql("SELECT count(*) FROM queue WHERE iif(?, name = ?, 1)", queue)[0][0]
+    run_sql("SELECT count(*) FROM queue WHERE iif(?1 IS NOT NULL, name = ?1, TRUE)", queue)[0][0]
   end
   
   # return the size of the queue file on disk
-  def size
-    run_sql("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")[0][0] 
-  end
+  #def size
+  #  run_sql("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")[0][0] 
+  #end
   
   def queues_info
-    run_sql("SELECT name, count(*) AS count, avg(unixepoch() - created_at), min(unixepoch() - created_at), max(unixepoch() - created_at) FROM queue GROUP BY name ORDER BY count DESC ")
+    run_stmt(:info)
   end
   
-  def info
-    counts = {}
+  def snapshot
+    queues = {}
     queues_info.each do |qc|
-      counts[qc[0]] = {count: qc[1], time_in_queue: {avg: qc[2], min: qc[3], max: qc[4]}}
+      #queues[qc[0]] = {count: qc[1], time_in_queue: {avg: qc[2], min: qc[3], max: qc[4]}}
+      queues[qc[0]] = qc[1]
     end
-    {size: size, count: count, info: counts}
+    {
+      summary: {
+        path: path,
+        journal_mode: journal_mode,
+        synchronous: synchronous,
+        size: size,
+        jobs: count
+      },
+      queues: queues      
+    }
   end
 
   private  
       
   def create_connection
-    conn = super
-    conn.wal_autocheckpoint = 10000 
-    sql = YAML.load_file("#{__dir__}/litequeue.sql.yml")
-    version = conn.get_first_value("PRAGMA user_version")
-    sql["schema"].each_pair do |v, obj| 
-      if v > version
-        conn.transaction(:immediate) do 
-          obj.each{|k, s| conn.execute(s)}
-          conn.user_version = v
+    super("#{__dir__}/litequeue.sql.yml") do |conn|
+      conn.wal_autocheckpoint = 10000 
+      # check if there is an old database and convert entries to the new format
+      if conn.get_first_value("select count(*) from sqlite_master where name = '_ul_queue_'") == 1
+        conn.transaction(:immediate) do
+          conn.execute("INSERT INTO queue(fire_at, name, value, created_at) SELECT fire_at, queue, value, created_at FROM _ul_queue_")
+          conn.execute("DROP TABLE _ul_queue_")
         end
       end
-    end 
-    sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
-    # check if there is an old database and convert entries to the new format
-    if conn.get_first_value("select count(*) from sqlite_master where name = '_ul_queue_'") == 1
-      conn.transaction(:immediate) do
-        conn.execute("INSERT INTO queue(fire_at, name, value, created_at) SELECT fire_at, queue, value, created_at FROM _ul_queue_")
-        conn.execute("DROP TABLE _ul_queue_")
-      end
     end
-    conn
   end
 
 end  

data/lib/litestack/litequeue.sql.yml

@@ -31,4 +31,15 @@ stmts:
     RETURNING id, value
   
   delete: DELETE FROM queue WHERE id = $1 RETURNING value
+  
+  info: >
+    SELECT 
+      name, 
+      count(*) AS count, 
+      avg(unixepoch() - created_at) AS avg, 
+      min(unixepoch() - created_at) AS min, 
+      max(unixepoch() - created_at) AS max
+    FROM queue 
+    GROUP BY name 
+    ORDER BY count DESC