comment_extractor 1.0.1

data/lib/comment_extractor/file.rb

@@ -0,0 +1,42 @@
+require 'comment_extractor/encoding'
+
+module CommentExtractor
+  module DetectableSchemeFile
+    THRESHOLD_BINARY = 0.3
+
+    refine File do
+      attr_accessor :content, :shebang
+
+      # [review] - How can I refine class method?
+      def File.shebang(path)
+        if File.extname(path).empty?
+          line = File.open(path) { |f| f.gets }
+          if /\A#!\s*(?<shebang_path>.+)/ =~ line
+            shebang_path
+          end
+        end
+      end
+
+      def File.binary?(file_path)
+        header = File.read(file_path, File.stat(file_path).blksize) || nil
+
+        if header.nil? || header.empty?
+          false
+        else
+          chars = header.chars
+          (chars.grep(' '..'~').size / chars.size.to_f) <= THRESHOLD_BINARY
+        end
+      end
+
+      def read_content
+        return if File.binary?(self.path)
+
+        if File.shebang(self.path)
+          self.gets # Remove shebang
+        end
+
+        CommentExtractor::Encoding.encode(self.read)
+      end
+    end
+  end
+end

data/lib/comment_extractor/parser.rb

@@ -0,0 +1,33 @@
+require 'comment_extractor/file'
+require 'comment_extractor/extractor_manager'
+
+using CommentExtractor::DetectableSchemeFile
+
+module CommentExtractor
+  class Parser
+    attr_accessor :extractor
+
+    def initialize(extractor)
+      @extractor = extractor
+    end
+
+    def extract_comments
+      @extractor.extract_comments
+    rescue NoMethodError
+      raise TypeError, "#{@extractor} should be a instance of #{Extractor}"
+    end
+
+    class << self
+      def for(file_path)
+        if extractor = ExtractorManager.can_extract(file_path)
+          content = File.open(file_path, 'r') { |f| f.read_content }
+
+          # Initialize parser
+          code_objects = CodeObjects.new(file: file_path)
+          instance_of_extractor = extractor.new(content, code_objects)
+          new(instance_of_extractor)
+        end
+      end
+    end
+  end
+end

data/lib/comment_extractor/smart_string_scanner.rb

@@ -0,0 +1,11 @@
+require 'strscan'
+
+module CommentExtractor
+  module SmartStringScanner
+    refine StringScanner do
+      def current_line
+        string[0...charpos].count("\n") + 1
+      end
+    end
+  end
+end

data/lib/comment_extractor/version.rb

@@ -0,0 +1,4 @@
+module CommentExtractor
+  Version = '1.0.1'
+  VERSION = Version
+end

data/lib/comment_extractor.rb

@@ -0,0 +1,18 @@
+require 'comment_extractor/code_object'
+require 'comment_extractor/configuration'
+require 'comment_extractor/encoding'
+require 'comment_extractor/extractor'
+require 'comment_extractor/extractor_manager'
+require 'comment_extractor/file'
+require 'comment_extractor/parser'
+require 'comment_extractor/version'
+
+module CommentExtractor
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.configure
+    yield(configuration)
+  end
+end

data/spec/assets/shebang_file

@@ -0,0 +1,3 @@
+#! /usr/local/bin/ruby
+1
+2

data/spec/assets/source_code/c.c

@@ -0,0 +1,158 @@
+//[-1-]Attribution-Share Alike 3.0 License, copyright (c) 2010 Pieter Hintjens, modified @alpaca-tc
+//[-2-] Copyright (c) 2010-2013 iMatix Corporation and Contributors
+//[-3-] Lets us build this source without creating a library
+#include "kvmsg.c"
+
+//[-6-]  zloop reactor handlers
+static int s_snapshots (zloop_t *loop, zmq_pollitem_t *poller, void *args);
+static int s_collector (zloop_t *loop, zmq_pollitem_t *poller, void *args);
+static int s_flush_ttl (zloop_t *loop, int timer_id, void *args);
+
+//[-11-]  Our server is defined by these properties
+typedef struct {
+    zctx_t *ctx;                //[-13-]  Context wrapper
+    zhash_t *kvmap;             //[-14-]  Key-value store
+    zloop_t *loop;              //[-15-]  zloop reactor
+    int port;                   //[-16-]  Main port we're working on
+    int64_t sequence;           //[-17-]  How many updates we're at
+    void *snapshot;             //[-18-]  Handle snapshot requests
+    void *publisher;            //[-19-]  Publish updates to clients
+    void *collector;            //[-20-]  Collect updates from clients
+} clonesrv_t;
+
+int main (void)
+{
+    clonesrv_t *self = (clonesrv_t *) zmalloc (sizeof (clonesrv_t));
+    self->port = 5556;
+    self->ctx = zctx_new ();
+    self->kvmap = zhash_new ();
+    self->loop = zloop_new ();
+    zloop_set_verbose (self->loop, false);
+
+    //[-32-]  Set up our clone server sockets
+    self->snapshot  = zsocket_new (self->ctx, ZMQ_ROUTER);
+    
+    self->publisher = zsocket_new (self->ctx, ZMQ_PUB);
+    zsocket_bind (self->publisher, "tcp://*:%d", self->port + 1);
+    self->collector = zsocket_new (self->ctx, ZMQ_PULL);
+    zsocket_bind (self->collector, "tcp://*:%d", self->port + 2);
+
+    //[-40-]  Register our handlers with reactor
+    zmq_pollitem_t poller = { 0, 0, ZMQ_POLLIN };
+    poller.socket = self->snapshot;
+    zloop_poller (self->loop, &poller, s_snapshots, self);
+    poller.socket = self->collector;
+    zloop_poller (self->loop, &poller, s_collector, self);
+    zloop_timer (self->loop, 1000, 0, s_flush_ttl, self);
+
+    //[-48-]  Run reactor until process interrupted
+    zloop_start (self->loop);
+
+    zloop_destroy (&self->loop);
+    zhash_destroy (&self->kvmap);
+    zctx_destroy (&self->ctx);
+    free (self);
+    return 0;
+}
+
+typedef struct {
+    void *socket;           //[-59-] ROUTER socket to send to
+    zframe_t *identity;     //[-60-]  Identity of peer who requested state
+    char *subtree;          //[-61-]  Client subtree specification
+} kvroute_t;
+
+static int
+s_send_single (const char *key, void *data, void *args)
+{
+    kvroute_t *kvroute = (kvroute_t *) args;
+    kvmsg_t *kvmsg = (kvmsg_t *) data;
+    if (strlen (kvroute->subtree) <= strlen (kvmsg_key (kvmsg))
+    &&  memcmp (kvroute->subtree,
+                kvmsg_key (kvmsg), strlen (kvroute->subtree)) == 0) {
+        zframe_send (&kvroute->identity,    //[-72-]  Choose recipient
+            kvroute->socket, ZFRAME_MORE + ZFRAME_REUSE);
+        kvmsg_send (kvmsg, kvroute->socket);
+    }
+    return 0;
+}
+
+static int
+s_snapshots (zloop_t *loop, zmq_pollitem_t *poller, void *args)
+{
+    clonesrv_t *self = (clonesrv_t *) args;
+
+    zframe_t *identity = zframe_recv (poller->socket);
+    if (identity) {
+        char *request = zstr_recv (poller->socket);
+        char *subtree = NULL;
+        if (streq (request, "ICANHAZ?")) {
+            free (request);
+            subtree = zstr_recv (poller->socket);
+        }
+        else
+            printf ("E: bad request, aborting\n");
+
+        if (subtree) {
+            kvroute_t routing = { poller->socket, identity, subtree };
+            zhash_foreach (self->kvmap, s_send_single, &routing);
+
+            zclock_log ("I: sending shapshot=%d", (int) self->sequence);
+            zframe_send (&identity, poller->socket, ZFRAME_MORE);
+            kvmsg_t *kvmsg = kvmsg_new (self->sequence);
+            kvmsg_set_key  (kvmsg, "KTHXBAI");
+            kvmsg_set_body (kvmsg, (byte *) subtree, 0);
+            kvmsg_send     (kvmsg, poller->socket);
+            kvmsg_destroy (&kvmsg);
+            free (subtree);
+        }
+        zframe_destroy(&identity);
+    }
+    return 0;
+}
+
+static int
+s_collector (zloop_t *loop, zmq_pollitem_t *poller, void *args)
+{
+    clonesrv_t *self = (clonesrv_t *) args;
+
+    kvmsg_t *kvmsg = kvmsg_recv (poller->socket);
+    if (kvmsg) {
+        kvmsg_set_sequence (kvmsg, ++self->sequence);
+        kvmsg_send (kvmsg, self->publisher);
+        int ttl = atoi (kvmsg_get_prop (kvmsg, "ttl"));
+        if (ttl)
+            kvmsg_set_prop (kvmsg, "ttl",
+                "%" PRId64, zclock_time () + ttl * 1000);
+        kvmsg_store (&kvmsg, self->kvmap);
+        zclock_log ("I: publishing update=%d", (int) self->sequence);
+    }
+    return 0;
+}
+
+//[-132-]  .split flush ephemeral values
+static int
+s_flush_single (const char *key, void *data, void *args)
+{
+    clonesrv_t *self = (clonesrv_t *) args;
+
+    kvmsg_t *kvmsg = (kvmsg_t *) data;
+    int64_t ttl;
+    sscanf (kvmsg_get_prop (kvmsg, "ttl"), "%" PRId64, &ttl);
+    if (ttl && zclock_time () >= ttl) {
+        kvmsg_set_sequence (kvmsg, ++self->sequence);
+        kvmsg_set_body (kvmsg, (byte *) "", 0);
+        kvmsg_send (kvmsg, self->publisher);
+        kvmsg_store (&kvmsg, self->kvmap);
+        zclock_log ("I: publishing delete=%d", (int) self->sequence);
+    }
+    return 0;
+}
+
+static int
+s_flush_ttl (zloop_t *loop, int timer_id, void *args)
+{
+    clonesrv_t *self = (clonesrv_t *) args;
+    if (self->kvmap)
+        zhash_foreach (self->kvmap, s_flush_single, args);
+    return 0;
+}

data/spec/assets/source_code/cc.cc

@@ -0,0 +1,24 @@
+//[-1-](C) 2013 Sebastian Mach (1983), this file is published under the terms of the
+//[-2-]GNU General Public License, Version 3 (a.k.a. GPLv3).
+//[-3-]See COPYING in the root-folder of the excygen project folder.
+#ifndef INTERSECTION_HH_INCLUDED_20130718
+#define INTERSECTION_HH_INCLUDED_20130718
+
+#include "DifferentialGeometry.hh"
+#include "Photometry/Material/Material.hh"
+#include <memory>
+
+namespace excyrender {
+
+    struct Intersection
+    {
+        DifferentialGeometry dg;
+        std::shared_ptr<const Photometry::Material::Material> material;
+    };
+
+    inline real distance(Intersection const &i) noexcept {
+        return i.dg.d;
+    } /*[-21-] multi line [-end-]*/
+}
+
+#endif //[-24-]INTERSECTION_HH_INCLUDED_20130718

data/spec/assets/source_code/clojure.clj

@@ -0,0 +1,41 @@
+(ns mtrelay
+(:refer-clojure :exclude [send])
+(:require [zhelpers :as mq])
+(:import [org.zeromq ZMQ$Context]))
+
+;;[-6-]Attribution-Share Alike 3.0 License, copyright (c) 2010 Pieter Hintjens, modified @alpaca-tc
+;;[-7-]Multithreaded relay in Clojure
+;;[-8-] Isaiah Peng <issaria@gmail.com>
+
+(defrecord Step1 [^ZMQ$Context ctx]
+Runnable
+(run [this]
+  (let [xmitter (mq/socket ctx mq/pair)]
+    (mq/connect xmitter "inproc://step2")
+    (println "Step 1 is ready, signaling step 2")
+    (mq/send xmitter "READY\u0000")
+    (.close xmitter))))
+
+(defrecord Step2 [^ZMQ$Context ctx]
+Runnable
+(run [this]
+  (let [receiver (mq/socket ctx mq/pair)
+        xmitter (mq/socket ctx mq/pair)]
+    (mq/bind receiver "inproc://step2")
+    (-> ctx Step1. Thread. .start)
+    ;;[-26-] Wait for signal and pass it on
+    (mq/recv-str receiver) ;;[-27-] Wait for signal and pass it on
+    (mq/connect xmitter "inproc://step3") ;;;;[-28-] Wait for signal and pass it on
+    (println "Step 2 is ready, signaling step 3") ;[-29-] Wait for signal and pass it on
+    (mq/send xmitter "READY\u0000")
+    (.close xmitter))))
+
+(defn -main []
+(let [ctx (mq/context 1)
+      receiver (mq/socket ctx mq/pair)]
+  (mq/bind receiver "inproc://step3")
+  (-> ctx Step2. Thread. .start)
+  (mq/recv-str receiver)
+  (.close receiver)
+  (println "Test successful!")
+  (.term ctx)))

data/spec/assets/source_code/coffee.coffee

@@ -0,0 +1,27 @@
+#[-1-] singleline
+class GiftModel
+  constructor: (gifts) -> #[-3-] singleline
+    @gifts = ko.observableArray gifts
+
+    @addGift = =>
+      @gifts.push({ name: "" })
+
+    @removeGift = (gift) =>
+      @gifts.remove gift
+
+      regexp = /// #[-12-] regexp comment
+      reg(exp) #[-13-] regexp comment
+      reg(exp) #[-14-] regexp comment
+      ///
+    @save = (form) =>
+      alert "Could now transmit to server: #{ko.utils.stringifyJson @gifts}"
+  ###[-18-] singleline
+[-19-] multi line
+###
+viewModel = new GiftModel(
+  [
+    { name: "Tall Hat", price: "39.95" } #[-23-] singleline
+        { name: "Long Cloak", price: "120.00"}
+  ])
+ko.applyBindings viewModel
+$("form").validate submitHandler: viewModel.save

data/spec/assets/source_code/cpp.cpp

@@ -0,0 +1,130 @@
+#include "zhelpers.hpp"
+#include <queue>
+//[-3-]Attribution-Share Alike 3.0 License, copyright (c) 2010 Pieter Hintjens, modified @alpaca-tc
+//[-4-]Basic request-reply client using REQ socket
+static void *
+client_thread (void *arg) {
+  zmq::context_t context(1);
+    zmq::socket_t client (context, ZMQ_REQ);
+    s_set_id (client);      //[-9-]Makes tracing easier
+    client.connect("ipc://frontend.ipc");
+
+    //[-12-]  Send request, get reply
+    s_send (client, "HELLO");
+    std::string reply = s_recv (client);
+    /*[-15-]  Send request, get reply [-end-]*/
+    return (NULL);
+}
+
+static void *
+worker_thread (void *arg) {
+  zmq::context_t context(1);
+    zmq::socket_t worker (context, ZMQ_REQ);
+    s_set_id (worker);
+    worker.connect("ipc://backend.ipc");
+
+    s_send (worker, "READY");
+
+    while (1) {
+        /*[-29-]
+[-30-]Read and save all frames until we get an empty frame
+[-31-]In this example there is only 1 but it could be more [-end-]*/
+        std::string address = s_recv (worker);
+        {
+            std::string empty = s_recv (worker);
+            assert (empty.size() == 0);
+        }
+
+        std::string request = s_recv (worker);
+        std::cout << "Worker: " << request << std::endl;
+
+        s_sendmore (worker, address);
+        s_sendmore (worker, "");
+        s_send     (worker, "OK");
+    }
+    return (NULL);
+}
+
+int main (int argc, char *argv[])
+{
+
+    zmq::context_t context(1);
+    zmq::socket_t frontend (context, ZMQ_ROUTER);
+    zmq::socket_t backend (context, ZMQ_ROUTER);
+    frontend.bind("ipc://frontend.ipc");
+    backend.bind("ipc://backend.ipc");
+
+    int client_nbr;
+    for (client_nbr = 0; client_nbr < 10; client_nbr++) {
+        pthread_t client;
+        pthread_create (&client, NULL, client_thread, NULL);
+    }
+    int worker_nbr;
+    for (worker_nbr = 0; worker_nbr < 3; worker_nbr++) {
+        pthread_t worker;
+        pthread_create (&worker, NULL, worker_thread, NULL);
+    }
+    std::queue<std::string> worker_queue;
+
+    while (1) {
+
+        zmq::pollitem_t items [] = {
+            { backend,  0, ZMQ_POLLIN, 0 },
+            { frontend, 0, ZMQ_POLLIN, 0 }
+        };
+        if (worker_queue.size())
+            zmq::poll (&items [0], 2, -1);
+        else
+            zmq::poll (&items [0], 1, -1);
+
+        if (items [0].revents & ZMQ_POLLIN) {
+
+            worker_queue.push(s_recv (backend));
+
+            {
+               std::string empty = s_recv (backend);
+               assert (empty.size() == 0);
+            }
+
+            std::string client_addr = s_recv (backend);
+
+            if (client_addr.compare("READY") != 0) {
+
+                {
+                    std::string empty = s_recv (backend);
+                    assert (empty.size() == 0);
+                }
+
+                std::string reply = s_recv (backend);
+                s_sendmore (frontend, client_addr);
+                s_sendmore (frontend, "");
+                s_send     (frontend, reply);
+
+                if (--client_nbr == 0)
+                    break;
+            }
+        }
+        if (items [1].revents & ZMQ_POLLIN) {
+
+            std::string client_addr = s_recv (frontend);
+
+            {
+                std::string empty = s_recv (frontend);
+                assert (empty.size() == 0);
+            }
+
+            std::string request = s_recv (frontend);
+
+            std::string worker_addr = worker_queue.front();
+            worker_queue.pop();
+
+            s_sendmore (backend, worker_addr);
+            s_sendmore (backend, "");
+            s_sendmore (backend, client_addr);
+            s_sendmore (backend, "");
+            s_send     (backend, request);
+        }
+    }
+    sleep (1);
+    return 0;
+}

data/spec/assets/source_code/cs.cs

@@ -0,0 +1,53 @@
+//[-1-]Attribution-Share Alike 3.0 License, copyright (c) 2010 Pieter Hintjens, modified @alpaca-tc
+//[-2-]  Author:     Yan Cui
+//[-3-]  Email:      theburningmonk@gmail.com
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using ZeroMQ;
+
+namespace zguide.clonesrv1
+{
+    internal class Program
+    {
+        public static void Main(string[] args)
+        {
+            using (var context = ZmqContext.Create())
+            {
+                using (var publisher = context.CreateSocket(SocketType.PUB))
+                {
+                    publisher.Bind("tcp://*:5556");
+                    Thread.Sleep(TimeSpan.FromMilliseconds(200));
+
+                    var interrupted = false; //[-23-] single line
+                    var sequence = 0L;/*[-24-] multi line [-end-]*/
+                    var random = new Random((int)DateTime.UtcNow.Ticks);
+                    var dict = new Dictionary<string, KvMsg>();
+                    /*[-27-]
+[-28-] multi line [-end-]
+*/
+                    Console.CancelKeyPress += (s, e) => { interrupted = true; };
+
+                    while (!interrupted)
+                    {
+                        sequence++;
+                        var kvmsg = new KvMsg(sequence);
+                        kvmsg.Key = random.Next(1, 10000).ToString();
+                        kvmsg.Body = random.Next(1, 1000000).ToString();
+
+                        kvmsg.Send(publisher);
+                        kvmsg.Store(dict);
+
+                        Console.WriteLine("Published {0}", kvmsg);
+
+                        Thread.Sleep(TimeSpan.FromMilliseconds(100));
+                    }
+
+                    Console.WriteLine(" Interrupted\n{0} messages out\n", sequence);
+                    Console.ReadKey();
+                }
+            }
+        }
+    }
+}

data/spec/assets/source_code/css.css

@@ -0,0 +1,37 @@
+@charset "UTF-8";
+@font-face {
+  font-family: ClumsyRegular;
+  src: url(/font/clumsy-webfont.eot);
+  src: local("☺"), url(/font/clumsy-webfont.woff) format("woff"), url(/font/clumsy-webfont.ttf) format("truetype"), url(/font/clumsy-webfont.svg#webfontYlE9Voyi) format("svg");
+  font-weight: normal;
+  font-style: normal;
+}
+/*[-9-] single comment [-end-]*/
+
+/*[-11-] multi line comment
+[-12-] multi line comment
+*/
+@font-face {
+  font-family: "Courier10BTRoman";
+  src: url(/font/courier-10-pitch-bt-webfont.eot);
+  src: local("☺"), url(/font/courier-10-pitch-bt-webfont.woff) format("woff"), url(/font/courier-10-pitch-bt-webfont.ttf) format("truetype"), url(/font/courier-10-pitch-bt-webfont.svg#webfontKRtE3TYu) format("svg");
+  font-weight: normal;
+  font-style: normal;
+}
+
+@font-face {
+  font-family: "LietzLindauHamburgRegular";
+  src: url(/font/lietzlindauhamburg-webfont.eot);
+  src: local("☺"), url(/font/lietzlindauhamburg-webfont.woff) format("woff"), url(/font/lietzlindauhamburg-webfont.ttf) format("truetype"), url(/font/lietzlindauhamburg-webfont.svg#webfont7wlycxIS) format("svg");
+  font-weight: normal;
+  font-style: normal;
+}
+
+.heading {
+  font-family: "PT Serif", "Georgia", "Helvetica Neue", Arial, sans-serif;
+}
+
+.sans, html .gist .gist-file .gist-meta {
+  font-family: "PT Sans", "Helvetica Neue", Arial, sans-serif;
+}
+